Datastore version one.
All checks were successful
Linting and tests / Tests (push) Successful in 59s
Linting and tests / Linting (push) Successful in 1m12s

Closes #2
This commit is contained in:
Stanislav Nikitin 2024-10-14 01:14:28 +05:00
parent bbb9824ad1
commit a901ba49e5
Signed by: pztrn
GPG Key ID: 1E944A0F0568B550
11 changed files with 571 additions and 9 deletions

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
*DS_Store* *DS_Store*
_build _build
server/localdev/featurer/data

50
docs/DATA_STORE.md Normal file
View File

@ -0,0 +1,50 @@
# Data storage
As of v0 (and probably v1) data storage implemented to be in plain text files. These files will be loaded into memory on startup and will be modified on request (e.g. when new flag is created). This is simplest approach for storing data.
Some limitations arises:
- Hard to implement application clustering (yet solvable).
- No reasonable security measures can be taken to ensure there will be no data loss. Do backups, please!
Until any of above lines aren't removed - limitation applies.
## Configuring data storage
Data storage can be configured via environment variables.
| Key | Format | Description |
| --- | --- | --- |
| `FEATURER_DATA_STORAGE_PATH` | | Path to root directory where Featurer data will reside. |
## Data storage structure
As Featurer is about applications, data storage was built around them.
Every application have own file with all environment and settings defined in it. These files contains JSON payload, so it is easy to save and load them, as well as backup.
Typical structure is:
```json
{
"version": "1",
"last_modified": "RFC3339 timestamp",
"data": {}
}
```
When other parts of Featurer asks to save value for some key, e.g. `production/feature_name/enabled`, it will be something like that when saved:
```json
{
"version": "1",
"last_modified": "2024-10-13T11:32:12.618218865Z",
"data": {
"production": {
"feature_name": {
"enabled": true
}
}
}
}
```

View File

@ -5,6 +5,7 @@ Welcome to Featurer's documentation!
## Administrator's corner ## Administrator's corner
- [Configuring HTTP servers](HTTP.md) - [Configuring HTTP servers](HTTP.md)
- [Configuring data storage](DATA_STORE.md)
## Users documentation ## Users documentation

View File

@ -2,14 +2,26 @@ package core
import "errors" import "errors"
// ServiceNameDatastore is a service name for data storage implementation. const (
const ServiceNameDatastore = "datastore" // DatastorePathDelimiter is a delimiter that should be used for path elements delimiter.
DatastorePathDelimiter = "/"
// ServiceNameDatastore is a service name for data storage implementation.
ServiceNameDatastore = "datastore"
)
var ( var (
// ErrDatastore indicates that error appeared somewhere in data storage service. // ErrDatastore indicates that error appeared somewhere in data storage service.
ErrDatastore = errors.New("datastore") ErrDatastore = errors.New("datastore")
// ErrDatastoreAppNotFound appears when requesting data for application which wasn't registered via API or loaded
// from disk.
ErrDatastoreAppNotFound = errors.New("application not found")
// ErrDatastoreServiceIsInvalid appears when data storage implementation isn't conforming to interface. // ErrDatastoreServiceIsInvalid appears when data storage implementation isn't conforming to interface.
ErrDatastoreServiceIsInvalid = errors.New("invalid datastore implementation") ErrDatastoreServiceIsInvalid = errors.New("invalid datastore implementation")
// ErrDatastoreValueInvalid appears when trying to get value from datastore using incompatible getter (e.g.
// when trying to get bool with GetString).
ErrDatastoreValueInvalid = errors.New("requested value is invalid")
// ErrDatastoreValueNotFound appears when trying to get value with path which isn't known.
ErrDatastoreValueNotFound = errors.New("requested value not found")
) )
// DataStore is an interface for data storage implementations. // DataStore is an interface for data storage implementations.

View File

@ -6,10 +6,15 @@ import (
"go.dev.pztrn.name/featurer/server/internal/application" "go.dev.pztrn.name/featurer/server/internal/application"
"go.dev.pztrn.name/featurer/server/internal/services/core" "go.dev.pztrn.name/featurer/server/internal/services/core"
"go.dev.pztrn.name/featurer/server/internal/services/core/datastore/models"
) )
var _ = core.DataStore(&datastore{})
type datastore struct { type datastore struct {
app *application.Application app *application.Application
storage map[string]*models.Application
dataPath string
} }
// Initialize initializes service. // Initialize initializes service.
@ -25,24 +30,33 @@ func Initialize(app *application.Application) error {
return nil return nil
} }
func (p *datastore) ConnectDependencies() error { func (d *datastore) ConnectDependencies() error {
return nil return nil
} }
func (p *datastore) GetName() string { func (d *datastore) GetName() string {
return core.ServiceNameDatastore return core.ServiceNameDatastore
} }
func (p *datastore) Initialize() error { func (d *datastore) Initialize() error {
slog.Info("Initializing data storage...") slog.Info("Initializing data storage...", "service", core.ServiceNameDatastore)
d.storage = make(map[string]*models.Application)
if err := d.initializePaths(); err != nil {
return fmt.Errorf("initialize: %w", err)
}
if err := d.loadData(); err != nil {
return fmt.Errorf("initialize: %w", err)
}
return nil return nil
} }
func (p *datastore) LaunchStartupTasks() error { func (d *datastore) LaunchStartupTasks() error {
return nil return nil
} }
func (p *datastore) Shutdown() error { func (d *datastore) Shutdown() error {
return nil return nil
} }

View File

@ -0,0 +1,182 @@
package datastore
import (
"fmt"
"log/slog"
"strings"
"go.dev.pztrn.name/featurer/server/internal/services/core"
)
func (d *datastore) get(appName, path string) (interface{}, error) {
appData, found := d.storage[appName]
if !found {
return nil, core.ErrDatastoreAppNotFound
}
pathSplitted := strings.Split(path, core.DatastorePathDelimiter)
value := appData.Data
for idx, pathPart := range pathSplitted {
slog.Debug(
"Checking path part to present in storage...",
"service", core.ServiceNameDatastore,
"path", path,
"path-part", pathPart,
)
newValueRaw, found := value[pathPart]
if !found {
slog.Debug(
"Path part not found!",
"service", core.ServiceNameDatastore,
"path", path,
"path-part", pathPart,
)
return nil, core.ErrDatastoreValueNotFound
}
slog.Debug(
"Path part found in storage. Will check if it is a map or final value.",
"service", core.ServiceNameDatastore,
"path", path,
"path-part", pathPart,
"value", newValueRaw,
)
newValueAsMap, validMap := newValueRaw.(map[string]interface{})
if !validMap && idx != len(pathSplitted)-1 {
slog.Debug(
"Value isn't map and it isn't last path part, returning error.",
"service", core.ServiceNameDatastore,
"path", path,
"path-part", pathPart,
"value", newValueRaw,
)
return nil, core.ErrDatastoreValueNotFound
}
value = newValueAsMap
if idx == len(pathSplitted)-1 {
slog.Debug(
"This is last path part, returning value.",
"service", core.ServiceNameDatastore,
"path", path,
"path-part", pathPart,
"value", value,
)
return newValueRaw, nil
}
}
return value, nil
}
func (d *datastore) GetBool(appName, path string) (bool, error) {
valRaw, err := d.get(appName, path)
if err != nil {
return false, err
}
slog.Debug(
"Got data, will try to convert to string",
"service", core.ServiceNameDatastore,
"path", appName+core.DatastorePathDelimiter+path,
"value", valRaw,
)
val, valid := valRaw.(bool)
if !valid {
return false, fmt.Errorf(
"%w: get bool for '%s': %w",
core.ErrDatastore,
appName+"/"+path,
core.ErrDatastoreValueInvalid,
)
}
return val, nil
}
func (d *datastore) GetInt64(appName, path string) (int64, error) {
valRaw, err := d.get(appName, path)
if err != nil {
return 0, err
}
slog.Debug(
"Got data, will try to convert to int64",
"service", core.ServiceNameDatastore,
"path", appName+core.DatastorePathDelimiter+path,
"value", valRaw,
)
val, valid := valRaw.(int64)
if !valid {
return 0, fmt.Errorf(
"%w: get int64 for '%s': %w",
core.ErrDatastore,
appName+"/"+path,
core.ErrDatastoreValueInvalid,
)
}
return val, nil
}
func (d *datastore) GetSliceOfStrings(appName, path string) ([]string, error) {
valRaw, err := d.get(appName, path)
if err != nil {
return nil, err
}
slog.Debug(
"Got data, will try to convert to slice of strings",
"service", core.ServiceNameDatastore,
"path", appName+core.DatastorePathDelimiter+path,
"value", valRaw,
)
val, valid := valRaw.([]string)
if !valid {
return nil, fmt.Errorf(
"%w: get slice of strings for '%s': %w",
core.ErrDatastore,
appName+"/"+path,
core.ErrDatastoreValueInvalid,
)
}
return val, nil
}
func (d *datastore) GetString(appName, path string) (string, error) {
valRaw, err := d.get(appName, path)
if err != nil {
return "", err
}
slog.Debug(
"Got data, will try to convert to string",
"service", core.ServiceNameDatastore,
"path", appName+core.DatastorePathDelimiter+path,
"value", valRaw,
)
val, valid := valRaw.(string)
if !valid {
return "", fmt.Errorf(
"%w: get string for '%s': %w",
core.ErrDatastore,
appName+"/"+path,
core.ErrDatastoreValueInvalid,
)
}
return val, nil
}

View File

@ -0,0 +1,10 @@
package models
import "time"
// Application is a data model for applications.
type Application struct {
Data map[string]interface{} `json:"data"`
LastModified time.Time `json:"last_modified"`
Version int64 `json:"version"`
}

View File

@ -0,0 +1,60 @@
package datastore
import (
"errors"
"fmt"
"log/slog"
"os"
"path/filepath"
"strings"
"go.dev.pztrn.name/featurer/server/internal/services/core"
)
const dataPathEnvKey = "FEATURER_DATA_STORAGE_PATH"
var errDataPathNotFoundInEnv = errors.New("FEATURER_DATA_STORAGE_PATH not defined")
// This function gets data path from environment and satinize it.
func (d *datastore) initializePaths() error {
dataPathRaw, found := os.LookupEnv(dataPathEnvKey)
if !found {
return fmt.Errorf("initialize data path: %w", errDataPathNotFoundInEnv)
}
dataPath, err := filepath.Abs(dataPathRaw)
if err != nil {
slog.Error(
"Unable to get absolute path!",
"service", core.ServiceNameDatastore,
"error", err.Error(),
)
return fmt.Errorf("initialize data path: get absolute path: %w", err)
}
homeDir, err := os.UserHomeDir()
if err != nil {
slog.Warn(
"Unable to get user's home directory path!",
"service", core.ServiceNameDatastore,
"error", err.Error(),
)
}
if homeDir != "" {
dataPath = strings.Replace(dataPath, "~", homeDir, 1)
}
dataPath = filepath.Clean(dataPath)
d.dataPath = dataPath
slog.Info(
"Got data path.",
"service", core.ServiceNameDatastore,
"path", d.dataPath,
)
return nil
}

View File

@ -0,0 +1,116 @@
package datastore
import (
"encoding/json"
"fmt"
"log/slog"
"os"
"path/filepath"
"strings"
"time"
"go.dev.pztrn.name/featurer/server/internal/services/core"
"go.dev.pztrn.name/featurer/server/internal/services/core/datastore/models"
)
func (d *datastore) Set(appName, path string, value interface{}) error {
// Set data in-memory.
slog.Debug(
"Set value for path",
"service", core.ServiceNameDatastore,
"application", appName,
"path", path,
"value", value,
)
pathSplitted := strings.Split(path, core.DatastorePathDelimiter)
appData, found := d.storage[appName]
if !found {
appData = &models.Application{
Version: 1,
LastModified: time.Now().UTC(),
Data: make(map[string]interface{}),
}
}
element := appData.Data
for idx, path := range pathSplitted {
if idx < len(pathSplitted)-1 {
e, found := element[path]
if !found {
mapElement := make(map[string]interface{})
element[path] = mapElement
element = mapElement
continue
}
// This element is a map[string]interface{} anyway.
theElement, _ := e.(map[string]interface{})
element[path] = theElement
element = theElement
continue
}
element[path] = value
}
// Immediately write to disk.
// ToDo: probably do it in another way?
slog.Debug(
"Writing data to disk",
"service", core.ServiceNameDatastore,
"application", appName,
"path", path,
"value", value,
)
appData.LastModified = time.Now().UTC()
bytes, err := json.Marshal(appData)
if err != nil {
slog.Error(
"Failed to marshal application data into JSON.",
"service", core.ServiceNameDatastore,
"application", appName,
"path", path,
"value", value,
"error", err.Error(),
)
return fmt.Errorf(
"%w: set value for '%s': marshal data: %w",
core.ErrDatastore,
appName+core.DatastorePathDelimiter+path,
err,
)
}
filePath := filepath.Join(d.dataPath, "applications", appName+".json")
// ToDo: fix permissions?
//nolint:gosec
if err := os.WriteFile(filePath, bytes, os.ModePerm); err != nil {
slog.Error(
"Failed to write application data into JSON.",
"service", core.ServiceNameDatastore,
"application", appName,
"path", path,
"value", value,
"error", err.Error(),
)
return fmt.Errorf(
"%w: set value for '%s': write data to disk: %w",
core.ErrDatastore,
appName+core.DatastorePathDelimiter+path,
err,
)
}
return nil
}

View File

@ -0,0 +1,113 @@
package datastore
import (
"encoding/json"
"fmt"
"io/fs"
"log/slog"
"os"
"path/filepath"
"strings"
"go.dev.pztrn.name/featurer/server/internal/services/core"
"go.dev.pztrn.name/featurer/server/internal/services/core/datastore/models"
)
func (d *datastore) loadData() error {
slog.Info(
"Loading data to memory...",
"service", core.ServiceNameDatastore,
)
filesToLoad := make([]string, 0)
if err := filepath.Walk(d.dataPath, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
slog.Debug(
"Adding file to list of files to load",
"service", core.ServiceNameDatastore,
"file", path,
)
filesToLoad = append(filesToLoad, path)
}
return nil
}); err != nil {
slog.Error(
"Failed to get recursive file listing.",
"service", core.ServiceNameDatastore,
"path", d.dataPath,
"error", err.Error(),
)
return fmt.Errorf("load data: get recursive file listing: %w", err)
}
slog.Debug(
"Got files to load",
"service", core.ServiceNameDatastore,
"files-count", len(filesToLoad),
"files", filesToLoad,
)
for _, file := range filesToLoad {
if err := d.loadFile(file); err != nil {
// Error logged in function.
return fmt.Errorf("load data: %w", err)
}
}
slog.Debug(
"Storage data loaded.",
"service", core.ServiceNameDatastore,
"storage-contents", d.storage,
)
return nil
}
func (d *datastore) loadFile(path string) error {
slog.Debug(
"Loading data from file...",
"service", core.ServiceNameDatastore,
"file", path,
)
// Here gosec whines about potential ".." in paths.
//nolint:gosec
data, err := os.ReadFile(path)
if err != nil {
slog.Error(
"Failed to read file data!",
"service", core.ServiceNameDatastore,
"file", path,
"error", err.Error(),
)
return fmt.Errorf("load file '%s': read file: %w", path, err)
}
appData := &models.Application{}
if err := json.Unmarshal(data, appData); err != nil {
slog.Error(
"Failed to parse file data!",
"service", core.ServiceNameDatastore,
"file", path,
"error", err.Error(),
)
return fmt.Errorf("load file '%s': parse file: %w", path, err)
}
// Application name is always in file's name and file's name is always has one dot.
appName := strings.Split(filepath.Base(path), ".")[0]
d.storage[appName] = appData
return nil
}

View File

@ -8,12 +8,15 @@ services:
ports: ports:
- "15000:5000" - "15000:5000"
- "15001:5001" - "15001:5001"
volumes:
- "./data:/data"
networks: networks:
featurer: featurer:
ipv4_address: 248.248.0.2 ipv4_address: 248.248.0.2
environment: environment:
FEATURER_API_SERVER_ADDRESS: "0.0.0.0:5000" FEATURER_API_SERVER_ADDRESS: "0.0.0.0:5000"
FEATURER_CMS_SERVER_ADDRESS: "0.0.0.0:5001" FEATURER_CMS_SERVER_ADDRESS: "0.0.0.0:5001"
FEATURER_DATA_STORAGE_PATH: "/data"
cap_add: cap_add:
- SYS_PTRACE - SYS_PTRACE