Datastore version one.

Closes #2

.gitignore

@@ -1,2 +1,3 @@
 *DS_Store*
 _build
+server/localdev/featurer/data

docs/DATA_STORE.md

@@ -0,0 +1,50 @@
+# Data storage
+
+As of v0 (and probably v1) data storage implemented to be in plain text files. These files will be loaded into memory on startup and will be modified on request (e.g. when new flag is created). This is simplest approach for storing data.
+
+Some limitations arises:
+
+- Hard to implement application clustering (yet solvable).
+- No reasonable security measures can be taken to ensure there will be no data loss. Do backups, please!
+
+Until any of above lines aren't removed - limitation applies.
+
+## Configuring data storage
+
+Data storage can be configured via environment variables.
+
+| Key | Format | Description |
+| --- | --- | --- |
+| `FEATURER_DATA_STORAGE_PATH` | | Path to root directory where Featurer data will reside. |
+
+## Data storage structure
+
+As Featurer is about applications, data storage was built around them.
+
+Every application have own file with all environment and settings defined in it. These files contains JSON payload, so it is easy to save and load them, as well as backup.
+
+Typical structure is:
+
+```json
+{
+    "version": "1",
+    "last_modified": "RFC3339 timestamp",
+    "data": {}
+}
+```
+
+When other parts of Featurer asks to save value for some key, e.g. `production/feature_name/enabled`, it will be something like that when saved:
+
+```json
+{
+    "version": "1",
+    "last_modified": "2024-10-13T11:32:12.618218865Z",
+    "data": {
+        "production": {
+            "feature_name": {
+                "enabled": true
+            }
+        }
+    }
+}
+```

docs/INDEX.md

@@ -5,6 +5,7 @@ Welcome to Featurer's documentation!
 ## Administrator's corner
 
 - [Configuring HTTP servers](HTTP.md)
+- [Configuring data storage](DATA_STORE.md)
 
 ## Users documentation
 

server/internal/services/core/datastore.go

@@ -2,14 +2,26 @@ package core
 
 import "errors"
 
+const (
+	// DatastorePathDelimiter is a delimiter that should be used for path elements delimiter.
+	DatastorePathDelimiter = "/"
 	// ServiceNameDatastore is a service name for data storage implementation.
-const ServiceNameDatastore = "datastore"
+	ServiceNameDatastore = "datastore"
+)
 
 var (
 	// ErrDatastore indicates that error appeared somewhere in data storage service.
 	ErrDatastore = errors.New("datastore")
+	// ErrDatastoreAppNotFound appears when requesting data for application which wasn't registered via API or loaded
+	// from disk.
+	ErrDatastoreAppNotFound = errors.New("application not found")
 	// ErrDatastoreServiceIsInvalid appears when data storage implementation isn't conforming to interface.
 	ErrDatastoreServiceIsInvalid = errors.New("invalid datastore implementation")
+	// ErrDatastoreValueInvalid appears when trying to get value from datastore using incompatible getter (e.g.
+	// when trying to get bool with GetString).
+	ErrDatastoreValueInvalid = errors.New("requested value is invalid")
+	// ErrDatastoreValueNotFound appears when trying to get value with path which isn't known.
+	ErrDatastoreValueNotFound = errors.New("requested value not found")
 )
 
 // DataStore is an interface for data storage implementations.

server/internal/services/core/datastore/datastore.go

@@ -6,10 +6,15 @@ import (
 
 	"go.dev.pztrn.name/featurer/server/internal/application"
 	"go.dev.pztrn.name/featurer/server/internal/services/core"
+	"go.dev.pztrn.name/featurer/server/internal/services/core/datastore/models"
 )
 
+var _ = core.DataStore(&datastore{})
+
 type datastore struct {
 	app      *application.Application
+	storage  map[string]*models.Application
+	dataPath string
 }
 
 // Initialize initializes service.
@@ -25,24 +30,33 @@ func Initialize(app *application.Application) error {
 	return nil
 }
 
-func (p *datastore) ConnectDependencies() error {
+func (d *datastore) ConnectDependencies() error {
 	return nil
 }
 
-func (p *datastore) GetName() string {
+func (d *datastore) GetName() string {
 	return core.ServiceNameDatastore
 }
 
-func (p *datastore) Initialize() error {
+func (d *datastore) Initialize() error {
-	slog.Info("Initializing data storage...")
+	slog.Info("Initializing data storage...", "service", core.ServiceNameDatastore)
+
+	d.storage = make(map[string]*models.Application)
+	if err := d.initializePaths(); err != nil {
+		return fmt.Errorf("initialize: %w", err)
+	}
+
+	if err := d.loadData(); err != nil {
+		return fmt.Errorf("initialize: %w", err)
+	}
 
 	return nil
 }
 
-func (p *datastore) LaunchStartupTasks() error {
+func (d *datastore) LaunchStartupTasks() error {
 	return nil
 }
 
-func (p *datastore) Shutdown() error {
+func (d *datastore) Shutdown() error {
 	return nil
 }

server/internal/services/core/datastore/get.go

@@ -0,0 +1,182 @@
+package datastore
+
+import (
+	"fmt"
+	"log/slog"
+	"strings"
+
+	"go.dev.pztrn.name/featurer/server/internal/services/core"
+)
+
+func (d *datastore) get(appName, path string) (interface{}, error) {
+	appData, found := d.storage[appName]
+	if !found {
+		return nil, core.ErrDatastoreAppNotFound
+	}
+
+	pathSplitted := strings.Split(path, core.DatastorePathDelimiter)
+
+	value := appData.Data
+
+	for idx, pathPart := range pathSplitted {
+		slog.Debug(
+			"Checking path part to present in storage...",
+			"service", core.ServiceNameDatastore,
+			"path", path,
+			"path-part", pathPart,
+		)
+
+		newValueRaw, found := value[pathPart]
+		if !found {
+			slog.Debug(
+				"Path part not found!",
+				"service", core.ServiceNameDatastore,
+				"path", path,
+				"path-part", pathPart,
+			)
+
+			return nil, core.ErrDatastoreValueNotFound
+		}
+
+		slog.Debug(
+			"Path part found in storage. Will check if it is a map or final value.",
+			"service", core.ServiceNameDatastore,
+			"path", path,
+			"path-part", pathPart,
+			"value", newValueRaw,
+		)
+
+		newValueAsMap, validMap := newValueRaw.(map[string]interface{})
+		if !validMap && idx != len(pathSplitted)-1 {
+			slog.Debug(
+				"Value isn't map and it isn't last path part, returning error.",
+				"service", core.ServiceNameDatastore,
+				"path", path,
+				"path-part", pathPart,
+				"value", newValueRaw,
+			)
+
+			return nil, core.ErrDatastoreValueNotFound
+		}
+
+		value = newValueAsMap
+
+		if idx == len(pathSplitted)-1 {
+			slog.Debug(
+				"This is last path part, returning value.",
+				"service", core.ServiceNameDatastore,
+				"path", path,
+				"path-part", pathPart,
+				"value", value,
+			)
+
+			return newValueRaw, nil
+		}
+	}
+
+	return value, nil
+}
+
+func (d *datastore) GetBool(appName, path string) (bool, error) {
+	valRaw, err := d.get(appName, path)
+	if err != nil {
+		return false, err
+	}
+
+	slog.Debug(
+		"Got data, will try to convert to string",
+		"service", core.ServiceNameDatastore,
+		"path", appName+core.DatastorePathDelimiter+path,
+		"value", valRaw,
+	)
+
+	val, valid := valRaw.(bool)
+	if !valid {
+		return false, fmt.Errorf(
+			"%w: get bool for '%s': %w",
+			core.ErrDatastore,
+			appName+"/"+path,
+			core.ErrDatastoreValueInvalid,
+		)
+	}
+
+	return val, nil
+}
+
+func (d *datastore) GetInt64(appName, path string) (int64, error) {
+	valRaw, err := d.get(appName, path)
+	if err != nil {
+		return 0, err
+	}
+
+	slog.Debug(
+		"Got data, will try to convert to int64",
+		"service", core.ServiceNameDatastore,
+		"path", appName+core.DatastorePathDelimiter+path,
+		"value", valRaw,
+	)
+
+	val, valid := valRaw.(int64)
+	if !valid {
+		return 0, fmt.Errorf(
+			"%w: get int64 for '%s': %w",
+			core.ErrDatastore,
+			appName+"/"+path,
+			core.ErrDatastoreValueInvalid,
+		)
+	}
+
+	return val, nil
+}
+
+func (d *datastore) GetSliceOfStrings(appName, path string) ([]string, error) {
+	valRaw, err := d.get(appName, path)
+	if err != nil {
+		return nil, err
+	}
+
+	slog.Debug(
+		"Got data, will try to convert to slice of strings",
+		"service", core.ServiceNameDatastore,
+		"path", appName+core.DatastorePathDelimiter+path,
+		"value", valRaw,
+	)
+
+	val, valid := valRaw.([]string)
+	if !valid {
+		return nil, fmt.Errorf(
+			"%w: get slice of strings for '%s': %w",
+			core.ErrDatastore,
+			appName+"/"+path,
+			core.ErrDatastoreValueInvalid,
+		)
+	}
+
+	return val, nil
+}
+
+func (d *datastore) GetString(appName, path string) (string, error) {
+	valRaw, err := d.get(appName, path)
+	if err != nil {
+		return "", err
+	}
+
+	slog.Debug(
+		"Got data, will try to convert to string",
+		"service", core.ServiceNameDatastore,
+		"path", appName+core.DatastorePathDelimiter+path,
+		"value", valRaw,
+	)
+
+	val, valid := valRaw.(string)
+	if !valid {
+		return "", fmt.Errorf(
+			"%w: get string for '%s': %w",
+			core.ErrDatastore,
+			appName+"/"+path,
+			core.ErrDatastoreValueInvalid,
+		)
+	}
+
+	return val, nil
+}

server/internal/services/core/datastore/models/application.go

@@ -0,0 +1,10 @@
+package models
+
+import "time"
+
+// Application is a data model for applications.
+type Application struct {
+	Data         map[string]interface{} `json:"data"`
+	LastModified time.Time              `json:"last_modified"`
+	Version      int64                  `json:"version"`
+}

server/internal/services/core/datastore/paths.go

@@ -0,0 +1,60 @@
+package datastore
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"go.dev.pztrn.name/featurer/server/internal/services/core"
+)
+
+const dataPathEnvKey = "FEATURER_DATA_STORAGE_PATH"
+
+var errDataPathNotFoundInEnv = errors.New("FEATURER_DATA_STORAGE_PATH not defined")
+
+// This function gets data path from environment and satinize it.
+func (d *datastore) initializePaths() error {
+	dataPathRaw, found := os.LookupEnv(dataPathEnvKey)
+	if !found {
+		return fmt.Errorf("initialize data path: %w", errDataPathNotFoundInEnv)
+	}
+
+	dataPath, err := filepath.Abs(dataPathRaw)
+	if err != nil {
+		slog.Error(
+			"Unable to get absolute path!",
+			"service", core.ServiceNameDatastore,
+			"error", err.Error(),
+		)
+
+		return fmt.Errorf("initialize data path: get absolute path: %w", err)
+	}
+
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		slog.Warn(
+			"Unable to get user's home directory path!",
+			"service", core.ServiceNameDatastore,
+			"error", err.Error(),
+		)
+	}
+
+	if homeDir != "" {
+		dataPath = strings.Replace(dataPath, "~", homeDir, 1)
+	}
+
+	dataPath = filepath.Clean(dataPath)
+
+	d.dataPath = dataPath
+
+	slog.Info(
+		"Got data path.",
+		"service", core.ServiceNameDatastore,
+		"path", d.dataPath,
+	)
+
+	return nil
+}

server/internal/services/core/datastore/set.go

@@ -0,0 +1,116 @@
+package datastore
+
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"go.dev.pztrn.name/featurer/server/internal/services/core"
+	"go.dev.pztrn.name/featurer/server/internal/services/core/datastore/models"
+)
+
+func (d *datastore) Set(appName, path string, value interface{}) error {
+	// Set data in-memory.
+	slog.Debug(
+		"Set value for path",
+		"service", core.ServiceNameDatastore,
+		"application", appName,
+		"path", path,
+		"value", value,
+	)
+
+	pathSplitted := strings.Split(path, core.DatastorePathDelimiter)
+
+	appData, found := d.storage[appName]
+	if !found {
+		appData = &models.Application{
+			Version:      1,
+			LastModified: time.Now().UTC(),
+			Data:         make(map[string]interface{}),
+		}
+	}
+
+	element := appData.Data
+
+	for idx, path := range pathSplitted {
+		if idx < len(pathSplitted)-1 {
+			e, found := element[path]
+			if !found {
+				mapElement := make(map[string]interface{})
+				element[path] = mapElement
+				element = mapElement
+
+				continue
+			}
+
+			// This element is a map[string]interface{} anyway.
+			theElement, _ := e.(map[string]interface{})
+
+			element[path] = theElement
+			element = theElement
+
+			continue
+		}
+
+		element[path] = value
+	}
+
+	// Immediately write to disk.
+	// ToDo: probably do it in another way?
+	slog.Debug(
+		"Writing data to disk",
+		"service", core.ServiceNameDatastore,
+		"application", appName,
+		"path", path,
+		"value", value,
+	)
+
+	appData.LastModified = time.Now().UTC()
+
+	bytes, err := json.Marshal(appData)
+	if err != nil {
+		slog.Error(
+			"Failed to marshal application data into JSON.",
+			"service", core.ServiceNameDatastore,
+			"application", appName,
+			"path", path,
+			"value", value,
+			"error", err.Error(),
+		)
+
+		return fmt.Errorf(
+			"%w: set value for '%s': marshal data: %w",
+			core.ErrDatastore,
+			appName+core.DatastorePathDelimiter+path,
+			err,
+		)
+	}
+
+	filePath := filepath.Join(d.dataPath, "applications", appName+".json")
+
+	// ToDo: fix permissions?
+	//nolint:gosec
+	if err := os.WriteFile(filePath, bytes, os.ModePerm); err != nil {
+		slog.Error(
+			"Failed to write application data into JSON.",
+			"service", core.ServiceNameDatastore,
+			"application", appName,
+			"path", path,
+			"value", value,
+			"error", err.Error(),
+		)
+
+		return fmt.Errorf(
+			"%w: set value for '%s': write data to disk: %w",
+			core.ErrDatastore,
+			appName+core.DatastorePathDelimiter+path,
+			err,
+		)
+	}
+
+	return nil
+}

server/internal/services/core/datastore/storage.go

@@ -0,0 +1,113 @@
+package datastore
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"go.dev.pztrn.name/featurer/server/internal/services/core"
+	"go.dev.pztrn.name/featurer/server/internal/services/core/datastore/models"
+)
+
+func (d *datastore) loadData() error {
+	slog.Info(
+		"Loading data to memory...",
+		"service", core.ServiceNameDatastore,
+	)
+
+	filesToLoad := make([]string, 0)
+
+	if err := filepath.Walk(d.dataPath, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !info.IsDir() {
+			slog.Debug(
+				"Adding file to list of files to load",
+				"service", core.ServiceNameDatastore,
+				"file", path,
+			)
+
+			filesToLoad = append(filesToLoad, path)
+		}
+
+		return nil
+	}); err != nil {
+		slog.Error(
+			"Failed to get recursive file listing.",
+			"service", core.ServiceNameDatastore,
+			"path", d.dataPath,
+			"error", err.Error(),
+		)
+
+		return fmt.Errorf("load data: get recursive file listing: %w", err)
+	}
+
+	slog.Debug(
+		"Got files to load",
+		"service", core.ServiceNameDatastore,
+		"files-count", len(filesToLoad),
+		"files", filesToLoad,
+	)
+
+	for _, file := range filesToLoad {
+		if err := d.loadFile(file); err != nil {
+			// Error logged in function.
+			return fmt.Errorf("load data:  %w", err)
+		}
+	}
+
+	slog.Debug(
+		"Storage data loaded.",
+		"service", core.ServiceNameDatastore,
+		"storage-contents", d.storage,
+	)
+
+	return nil
+}
+
+func (d *datastore) loadFile(path string) error {
+	slog.Debug(
+		"Loading data from file...",
+		"service", core.ServiceNameDatastore,
+		"file", path,
+	)
+
+	// Here gosec whines about potential ".." in paths.
+	//nolint:gosec
+	data, err := os.ReadFile(path)
+	if err != nil {
+		slog.Error(
+			"Failed to read file data!",
+			"service", core.ServiceNameDatastore,
+			"file", path,
+			"error", err.Error(),
+		)
+
+		return fmt.Errorf("load file '%s': read file: %w", path, err)
+	}
+
+	appData := &models.Application{}
+
+	if err := json.Unmarshal(data, appData); err != nil {
+		slog.Error(
+			"Failed to parse file data!",
+			"service", core.ServiceNameDatastore,
+			"file", path,
+			"error", err.Error(),
+		)
+
+		return fmt.Errorf("load file '%s': parse file: %w", path, err)
+	}
+
+	// Application name is always in file's name and file's name is always has one dot.
+	appName := strings.Split(filepath.Base(path), ".")[0]
+	d.storage[appName] = appData
+
+	return nil
+}

server/localdev/featurer/docker-compose.yaml

@@ -8,12 +8,15 @@ services:
     ports:
       - "15000:5000"
       - "15001:5001"
+    volumes:
+      - "./data:/data"
     networks:
       featurer:
         ipv4_address: 248.248.0.2
     environment:
       FEATURER_API_SERVER_ADDRESS: "0.0.0.0:5000"
       FEATURER_CMS_SERVER_ADDRESS: "0.0.0.0:5001"
+      FEATURER_DATA_STORAGE_PATH: "/data"
     cap_add:
       - SYS_PTRACE