From a901ba49e5a257cb9cf39b35308e0c0ed9342113 Mon Sep 17 00:00:00 2001 From: "Stanislav N. aka pztrn" Date: Mon, 14 Oct 2024 01:14:28 +0500 Subject: [PATCH] Datastore version one. Closes #2 --- .gitignore | 1 + docs/DATA_STORE.md | 50 +++++ docs/INDEX.md | 1 + server/internal/services/core/datastore.go | 16 +- .../services/core/datastore/datastore.go | 28 ++- .../internal/services/core/datastore/get.go | 182 ++++++++++++++++++ .../core/datastore/models/application.go | 10 + .../internal/services/core/datastore/paths.go | 60 ++++++ .../internal/services/core/datastore/set.go | 116 +++++++++++ .../services/core/datastore/storage.go | 113 +++++++++++ server/localdev/featurer/docker-compose.yaml | 3 + 11 files changed, 571 insertions(+), 9 deletions(-) create mode 100644 docs/DATA_STORE.md create mode 100644 server/internal/services/core/datastore/get.go create mode 100644 server/internal/services/core/datastore/models/application.go create mode 100644 server/internal/services/core/datastore/paths.go create mode 100644 server/internal/services/core/datastore/set.go create mode 100644 server/internal/services/core/datastore/storage.go diff --git a/.gitignore b/.gitignore index 83c3088..a952b12 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *DS_Store* _build +server/localdev/featurer/data diff --git a/docs/DATA_STORE.md b/docs/DATA_STORE.md new file mode 100644 index 0000000..b4e1be1 --- /dev/null +++ b/docs/DATA_STORE.md @@ -0,0 +1,50 @@ +# Data storage + +As of v0 (and probably v1) data storage implemented to be in plain text files. These files will be loaded into memory on startup and will be modified on request (e.g. when new flag is created). This is simplest approach for storing data. + +Some limitations arises: + +- Hard to implement application clustering (yet solvable). +- No reasonable security measures can be taken to ensure there will be no data loss. Do backups, please! + +Until any of above lines aren't removed - limitation applies. + +## Configuring data storage + +Data storage can be configured via environment variables. + +| Key | Format | Description | +| --- | --- | --- | +| `FEATURER_DATA_STORAGE_PATH` | | Path to root directory where Featurer data will reside. | + +## Data storage structure + +As Featurer is about applications, data storage was built around them. + +Every application have own file with all environment and settings defined in it. These files contains JSON payload, so it is easy to save and load them, as well as backup. + +Typical structure is: + +```json +{ + "version": "1", + "last_modified": "RFC3339 timestamp", + "data": {} +} +``` + +When other parts of Featurer asks to save value for some key, e.g. `production/feature_name/enabled`, it will be something like that when saved: + +```json +{ + "version": "1", + "last_modified": "2024-10-13T11:32:12.618218865Z", + "data": { + "production": { + "feature_name": { + "enabled": true + } + } + } +} +``` diff --git a/docs/INDEX.md b/docs/INDEX.md index fb57540..5f71e89 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -5,6 +5,7 @@ Welcome to Featurer's documentation! ## Administrator's corner - [Configuring HTTP servers](HTTP.md) +- [Configuring data storage](DATA_STORE.md) ## Users documentation diff --git a/server/internal/services/core/datastore.go b/server/internal/services/core/datastore.go index 77ac53b..46d6a69 100644 --- a/server/internal/services/core/datastore.go +++ b/server/internal/services/core/datastore.go @@ -2,14 +2,26 @@ package core import "errors" -// ServiceNameDatastore is a service name for data storage implementation. -const ServiceNameDatastore = "datastore" +const ( + // DatastorePathDelimiter is a delimiter that should be used for path elements delimiter. + DatastorePathDelimiter = "/" + // ServiceNameDatastore is a service name for data storage implementation. + ServiceNameDatastore = "datastore" +) var ( // ErrDatastore indicates that error appeared somewhere in data storage service. ErrDatastore = errors.New("datastore") + // ErrDatastoreAppNotFound appears when requesting data for application which wasn't registered via API or loaded + // from disk. + ErrDatastoreAppNotFound = errors.New("application not found") // ErrDatastoreServiceIsInvalid appears when data storage implementation isn't conforming to interface. ErrDatastoreServiceIsInvalid = errors.New("invalid datastore implementation") + // ErrDatastoreValueInvalid appears when trying to get value from datastore using incompatible getter (e.g. + // when trying to get bool with GetString). + ErrDatastoreValueInvalid = errors.New("requested value is invalid") + // ErrDatastoreValueNotFound appears when trying to get value with path which isn't known. + ErrDatastoreValueNotFound = errors.New("requested value not found") ) // DataStore is an interface for data storage implementations. diff --git a/server/internal/services/core/datastore/datastore.go b/server/internal/services/core/datastore/datastore.go index 84d6579..b427005 100644 --- a/server/internal/services/core/datastore/datastore.go +++ b/server/internal/services/core/datastore/datastore.go @@ -6,10 +6,15 @@ import ( "go.dev.pztrn.name/featurer/server/internal/application" "go.dev.pztrn.name/featurer/server/internal/services/core" + "go.dev.pztrn.name/featurer/server/internal/services/core/datastore/models" ) +var _ = core.DataStore(&datastore{}) + type datastore struct { - app *application.Application + app *application.Application + storage map[string]*models.Application + dataPath string } // Initialize initializes service. @@ -25,24 +30,33 @@ func Initialize(app *application.Application) error { return nil } -func (p *datastore) ConnectDependencies() error { +func (d *datastore) ConnectDependencies() error { return nil } -func (p *datastore) GetName() string { +func (d *datastore) GetName() string { return core.ServiceNameDatastore } -func (p *datastore) Initialize() error { - slog.Info("Initializing data storage...") +func (d *datastore) Initialize() error { + slog.Info("Initializing data storage...", "service", core.ServiceNameDatastore) + + d.storage = make(map[string]*models.Application) + if err := d.initializePaths(); err != nil { + return fmt.Errorf("initialize: %w", err) + } + + if err := d.loadData(); err != nil { + return fmt.Errorf("initialize: %w", err) + } return nil } -func (p *datastore) LaunchStartupTasks() error { +func (d *datastore) LaunchStartupTasks() error { return nil } -func (p *datastore) Shutdown() error { +func (d *datastore) Shutdown() error { return nil } diff --git a/server/internal/services/core/datastore/get.go b/server/internal/services/core/datastore/get.go new file mode 100644 index 0000000..dee4bb4 --- /dev/null +++ b/server/internal/services/core/datastore/get.go @@ -0,0 +1,182 @@ +package datastore + +import ( + "fmt" + "log/slog" + "strings" + + "go.dev.pztrn.name/featurer/server/internal/services/core" +) + +func (d *datastore) get(appName, path string) (interface{}, error) { + appData, found := d.storage[appName] + if !found { + return nil, core.ErrDatastoreAppNotFound + } + + pathSplitted := strings.Split(path, core.DatastorePathDelimiter) + + value := appData.Data + + for idx, pathPart := range pathSplitted { + slog.Debug( + "Checking path part to present in storage...", + "service", core.ServiceNameDatastore, + "path", path, + "path-part", pathPart, + ) + + newValueRaw, found := value[pathPart] + if !found { + slog.Debug( + "Path part not found!", + "service", core.ServiceNameDatastore, + "path", path, + "path-part", pathPart, + ) + + return nil, core.ErrDatastoreValueNotFound + } + + slog.Debug( + "Path part found in storage. Will check if it is a map or final value.", + "service", core.ServiceNameDatastore, + "path", path, + "path-part", pathPart, + "value", newValueRaw, + ) + + newValueAsMap, validMap := newValueRaw.(map[string]interface{}) + if !validMap && idx != len(pathSplitted)-1 { + slog.Debug( + "Value isn't map and it isn't last path part, returning error.", + "service", core.ServiceNameDatastore, + "path", path, + "path-part", pathPart, + "value", newValueRaw, + ) + + return nil, core.ErrDatastoreValueNotFound + } + + value = newValueAsMap + + if idx == len(pathSplitted)-1 { + slog.Debug( + "This is last path part, returning value.", + "service", core.ServiceNameDatastore, + "path", path, + "path-part", pathPart, + "value", value, + ) + + return newValueRaw, nil + } + } + + return value, nil +} + +func (d *datastore) GetBool(appName, path string) (bool, error) { + valRaw, err := d.get(appName, path) + if err != nil { + return false, err + } + + slog.Debug( + "Got data, will try to convert to string", + "service", core.ServiceNameDatastore, + "path", appName+core.DatastorePathDelimiter+path, + "value", valRaw, + ) + + val, valid := valRaw.(bool) + if !valid { + return false, fmt.Errorf( + "%w: get bool for '%s': %w", + core.ErrDatastore, + appName+"/"+path, + core.ErrDatastoreValueInvalid, + ) + } + + return val, nil +} + +func (d *datastore) GetInt64(appName, path string) (int64, error) { + valRaw, err := d.get(appName, path) + if err != nil { + return 0, err + } + + slog.Debug( + "Got data, will try to convert to int64", + "service", core.ServiceNameDatastore, + "path", appName+core.DatastorePathDelimiter+path, + "value", valRaw, + ) + + val, valid := valRaw.(int64) + if !valid { + return 0, fmt.Errorf( + "%w: get int64 for '%s': %w", + core.ErrDatastore, + appName+"/"+path, + core.ErrDatastoreValueInvalid, + ) + } + + return val, nil +} + +func (d *datastore) GetSliceOfStrings(appName, path string) ([]string, error) { + valRaw, err := d.get(appName, path) + if err != nil { + return nil, err + } + + slog.Debug( + "Got data, will try to convert to slice of strings", + "service", core.ServiceNameDatastore, + "path", appName+core.DatastorePathDelimiter+path, + "value", valRaw, + ) + + val, valid := valRaw.([]string) + if !valid { + return nil, fmt.Errorf( + "%w: get slice of strings for '%s': %w", + core.ErrDatastore, + appName+"/"+path, + core.ErrDatastoreValueInvalid, + ) + } + + return val, nil +} + +func (d *datastore) GetString(appName, path string) (string, error) { + valRaw, err := d.get(appName, path) + if err != nil { + return "", err + } + + slog.Debug( + "Got data, will try to convert to string", + "service", core.ServiceNameDatastore, + "path", appName+core.DatastorePathDelimiter+path, + "value", valRaw, + ) + + val, valid := valRaw.(string) + if !valid { + return "", fmt.Errorf( + "%w: get string for '%s': %w", + core.ErrDatastore, + appName+"/"+path, + core.ErrDatastoreValueInvalid, + ) + } + + return val, nil +} diff --git a/server/internal/services/core/datastore/models/application.go b/server/internal/services/core/datastore/models/application.go new file mode 100644 index 0000000..a1def23 --- /dev/null +++ b/server/internal/services/core/datastore/models/application.go @@ -0,0 +1,10 @@ +package models + +import "time" + +// Application is a data model for applications. +type Application struct { + Data map[string]interface{} `json:"data"` + LastModified time.Time `json:"last_modified"` + Version int64 `json:"version"` +} diff --git a/server/internal/services/core/datastore/paths.go b/server/internal/services/core/datastore/paths.go new file mode 100644 index 0000000..df61f6c --- /dev/null +++ b/server/internal/services/core/datastore/paths.go @@ -0,0 +1,60 @@ +package datastore + +import ( + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + + "go.dev.pztrn.name/featurer/server/internal/services/core" +) + +const dataPathEnvKey = "FEATURER_DATA_STORAGE_PATH" + +var errDataPathNotFoundInEnv = errors.New("FEATURER_DATA_STORAGE_PATH not defined") + +// This function gets data path from environment and satinize it. +func (d *datastore) initializePaths() error { + dataPathRaw, found := os.LookupEnv(dataPathEnvKey) + if !found { + return fmt.Errorf("initialize data path: %w", errDataPathNotFoundInEnv) + } + + dataPath, err := filepath.Abs(dataPathRaw) + if err != nil { + slog.Error( + "Unable to get absolute path!", + "service", core.ServiceNameDatastore, + "error", err.Error(), + ) + + return fmt.Errorf("initialize data path: get absolute path: %w", err) + } + + homeDir, err := os.UserHomeDir() + if err != nil { + slog.Warn( + "Unable to get user's home directory path!", + "service", core.ServiceNameDatastore, + "error", err.Error(), + ) + } + + if homeDir != "" { + dataPath = strings.Replace(dataPath, "~", homeDir, 1) + } + + dataPath = filepath.Clean(dataPath) + + d.dataPath = dataPath + + slog.Info( + "Got data path.", + "service", core.ServiceNameDatastore, + "path", d.dataPath, + ) + + return nil +} diff --git a/server/internal/services/core/datastore/set.go b/server/internal/services/core/datastore/set.go new file mode 100644 index 0000000..3d08693 --- /dev/null +++ b/server/internal/services/core/datastore/set.go @@ -0,0 +1,116 @@ +package datastore + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "go.dev.pztrn.name/featurer/server/internal/services/core" + "go.dev.pztrn.name/featurer/server/internal/services/core/datastore/models" +) + +func (d *datastore) Set(appName, path string, value interface{}) error { + // Set data in-memory. + slog.Debug( + "Set value for path", + "service", core.ServiceNameDatastore, + "application", appName, + "path", path, + "value", value, + ) + + pathSplitted := strings.Split(path, core.DatastorePathDelimiter) + + appData, found := d.storage[appName] + if !found { + appData = &models.Application{ + Version: 1, + LastModified: time.Now().UTC(), + Data: make(map[string]interface{}), + } + } + + element := appData.Data + + for idx, path := range pathSplitted { + if idx < len(pathSplitted)-1 { + e, found := element[path] + if !found { + mapElement := make(map[string]interface{}) + element[path] = mapElement + element = mapElement + + continue + } + + // This element is a map[string]interface{} anyway. + theElement, _ := e.(map[string]interface{}) + + element[path] = theElement + element = theElement + + continue + } + + element[path] = value + } + + // Immediately write to disk. + // ToDo: probably do it in another way? + slog.Debug( + "Writing data to disk", + "service", core.ServiceNameDatastore, + "application", appName, + "path", path, + "value", value, + ) + + appData.LastModified = time.Now().UTC() + + bytes, err := json.Marshal(appData) + if err != nil { + slog.Error( + "Failed to marshal application data into JSON.", + "service", core.ServiceNameDatastore, + "application", appName, + "path", path, + "value", value, + "error", err.Error(), + ) + + return fmt.Errorf( + "%w: set value for '%s': marshal data: %w", + core.ErrDatastore, + appName+core.DatastorePathDelimiter+path, + err, + ) + } + + filePath := filepath.Join(d.dataPath, "applications", appName+".json") + + // ToDo: fix permissions? + //nolint:gosec + if err := os.WriteFile(filePath, bytes, os.ModePerm); err != nil { + slog.Error( + "Failed to write application data into JSON.", + "service", core.ServiceNameDatastore, + "application", appName, + "path", path, + "value", value, + "error", err.Error(), + ) + + return fmt.Errorf( + "%w: set value for '%s': write data to disk: %w", + core.ErrDatastore, + appName+core.DatastorePathDelimiter+path, + err, + ) + } + + return nil +} diff --git a/server/internal/services/core/datastore/storage.go b/server/internal/services/core/datastore/storage.go new file mode 100644 index 0000000..8a86b01 --- /dev/null +++ b/server/internal/services/core/datastore/storage.go @@ -0,0 +1,113 @@ +package datastore + +import ( + "encoding/json" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" + + "go.dev.pztrn.name/featurer/server/internal/services/core" + "go.dev.pztrn.name/featurer/server/internal/services/core/datastore/models" +) + +func (d *datastore) loadData() error { + slog.Info( + "Loading data to memory...", + "service", core.ServiceNameDatastore, + ) + + filesToLoad := make([]string, 0) + + if err := filepath.Walk(d.dataPath, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.IsDir() { + slog.Debug( + "Adding file to list of files to load", + "service", core.ServiceNameDatastore, + "file", path, + ) + + filesToLoad = append(filesToLoad, path) + } + + return nil + }); err != nil { + slog.Error( + "Failed to get recursive file listing.", + "service", core.ServiceNameDatastore, + "path", d.dataPath, + "error", err.Error(), + ) + + return fmt.Errorf("load data: get recursive file listing: %w", err) + } + + slog.Debug( + "Got files to load", + "service", core.ServiceNameDatastore, + "files-count", len(filesToLoad), + "files", filesToLoad, + ) + + for _, file := range filesToLoad { + if err := d.loadFile(file); err != nil { + // Error logged in function. + return fmt.Errorf("load data: %w", err) + } + } + + slog.Debug( + "Storage data loaded.", + "service", core.ServiceNameDatastore, + "storage-contents", d.storage, + ) + + return nil +} + +func (d *datastore) loadFile(path string) error { + slog.Debug( + "Loading data from file...", + "service", core.ServiceNameDatastore, + "file", path, + ) + + // Here gosec whines about potential ".." in paths. + //nolint:gosec + data, err := os.ReadFile(path) + if err != nil { + slog.Error( + "Failed to read file data!", + "service", core.ServiceNameDatastore, + "file", path, + "error", err.Error(), + ) + + return fmt.Errorf("load file '%s': read file: %w", path, err) + } + + appData := &models.Application{} + + if err := json.Unmarshal(data, appData); err != nil { + slog.Error( + "Failed to parse file data!", + "service", core.ServiceNameDatastore, + "file", path, + "error", err.Error(), + ) + + return fmt.Errorf("load file '%s': parse file: %w", path, err) + } + + // Application name is always in file's name and file's name is always has one dot. + appName := strings.Split(filepath.Base(path), ".")[0] + d.storage[appName] = appData + + return nil +} diff --git a/server/localdev/featurer/docker-compose.yaml b/server/localdev/featurer/docker-compose.yaml index 7011fe2..7c006ad 100644 --- a/server/localdev/featurer/docker-compose.yaml +++ b/server/localdev/featurer/docker-compose.yaml @@ -8,12 +8,15 @@ services: ports: - "15000:5000" - "15001:5001" + volumes: + - "./data:/data" networks: featurer: ipv4_address: 248.248.0.2 environment: FEATURER_API_SERVER_ADDRESS: "0.0.0.0:5000" FEATURER_CMS_SERVER_ADDRESS: "0.0.0.0:5001" + FEATURER_DATA_STORAGE_PATH: "/data" cap_add: - SYS_PTRACE