From f915470c0bf511e602d7cdb0b32cca4ef3584763 Mon Sep 17 00:00:00 2001 From: "Stanislav N. aka pztrn" Date: Sun, 29 Nov 2020 03:22:39 +0500 Subject: [PATCH] Rethink how metricator will work, golangci-lint and gitlab ci configs. --- .gitignore | 1 + .gitlab-ci.yml | 10 +++ .golangci.yml | 23 ++++++ Makefile | 4 +- README.md | 5 +- cmd/metricatord/main.go | 26 +++++-- go.mod | 2 +- go.sum | 2 + internal/application/application.go | 63 ++++++++++++++++ internal/application/config.go | 15 ++++ internal/application/fetcher.go | 4 + internal/common/{const.go => version_info.go} | 0 internal/configuration/config.go | 20 +---- internal/datastore/application_store.go | 25 ------- internal/datastore/datastore.go | 56 -------------- internal/httpserver/httpserver.go | 2 + internal/storage/generic.go | 15 ++++ internal/storage/memory/memory.go | 75 +++++++++++++++++++ internal/storage/metrics.go | 6 ++ metricator.example.yaml | 14 +++- 20 files changed, 256 insertions(+), 112 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100644 .golangci.yml create mode 100644 internal/application/application.go create mode 100644 internal/application/config.go create mode 100644 internal/application/fetcher.go rename internal/common/{const.go => version_info.go} (100%) delete mode 100644 internal/datastore/application_store.go delete mode 100644 internal/datastore/datastore.go create mode 100644 internal/storage/generic.go create mode 100644 internal/storage/memory/memory.go create mode 100644 internal/storage/metrics.go diff --git a/.gitignore b/.gitignore index 6e17a58..476f2a0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ ._bin .vscode +metricator.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..591304a --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,10 @@ +stages: + - test + +lint: + stage: test + tags: + - docker + image: golangci/golangci-lint:v1.32.2 + script: + - golangci-lint run diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..cc70430 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,23 @@ +run: + deadline: 5m +linters: + enable-all: true + disable: + # Because globals might exist, but according to our codestyle they + # should be lowercased and considered as unexported. + - gochecknoglobals + # While it might be useful it'll create more problems that will solve. + - gocritic + # Complains about main() lengths, which isn't an issue. + - funlen + # Magic numbers might be everywhere. Disabled for now. + - gomnd + # ToDos everywhere + - godox + # Why? WHY? WHY _test??? + - testpackage +linters-settings: + lll: + line-length: 120 + gocyclo: + min-complexity: 40 diff --git a/Makefile b/Makefile index a9dd857..d997f64 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ # This is a Metricator Makefile. # It contains calls to scripts placed in scripts directory. +CONFIG ?= ./metricator.example.yaml + help: Makefile @echo -e "Metricator Makefile available subcommands:\n" @cat $< | grep "## " | sort | sed -n 's/^## //p' @@ -19,7 +21,7 @@ metricatord-build: check-build-dir ## metricatord-run: starts metricator daemon. metricatord-run: metricatord-build - ./._bin/metricatord -config ./metricator.example.yaml + ./._bin/metricatord -config ${CONFIG} show-git-data: @echo "Parameters for current source code state:" diff --git a/README.md b/README.md index 479c7f0..9c91eaf 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,11 @@ Metricator will issue only one request and cache data in memory between them. Al ## Installation -*TBW* +TBW ## Configuration -*TBW* - +TBW ## Documentation diff --git a/cmd/metricatord/main.go b/cmd/metricatord/main.go index 3a53d33..d75e4ac 100644 --- a/cmd/metricatord/main.go +++ b/cmd/metricatord/main.go @@ -8,9 +8,9 @@ import ( "os/signal" "syscall" + "go.dev.pztrn.name/metricator/internal/application" "go.dev.pztrn.name/metricator/internal/common" "go.dev.pztrn.name/metricator/internal/configuration" - "go.dev.pztrn.name/metricator/internal/datastore" "go.dev.pztrn.name/metricator/internal/httpserver" ) @@ -26,16 +26,27 @@ func main() { config := configuration.NewConfig() httpSrv, httpStopped := httpserver.NewHTTPServer(mainCtx, config) - dataStore, dataStoreStopped := datastore.NewDataStore(mainCtx, config) + // Parse configuration. flag.Parse() + err := config.Parse() if err != nil { log.Fatalln("Failed to parse configuration:", err.Error()) } + log.Printf("Configuration parsed: %+v\n", config) - dataStore.Start() + // Create applications. + apps := make([]*application.Application, 0, len(config.Applications)) + + for appName, appConfig := range config.Applications { + app := application.NewApplication(mainCtx, appName, appConfig) + app.Start() + + apps = append(apps, app) + } + httpSrv.Start() log.Println("Metricator is started and ready to serve requests") @@ -46,15 +57,18 @@ func main() { signal.Notify(signalHandler, os.Interrupt, syscall.SIGTERM) - go func() { + go func(apps []*application.Application) { <-signalHandler cancelFunc() - <-dataStoreStopped + for _, app := range apps { + <-app.GetDoneChan() + } + <-httpStopped shutdownDone <- true - }() + }(apps) <-shutdownDone log.Println("Metricator stopped") diff --git a/go.mod b/go.mod index 7f39460..38bfd26 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,4 @@ module go.dev.pztrn.name/metricator go 1.13 -require gopkg.in/yaml.v2 v2.3.0 +require gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 8fabe8d..976708a 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/internal/application/application.go b/internal/application/application.go new file mode 100644 index 0000000..f44a6d0 --- /dev/null +++ b/internal/application/application.go @@ -0,0 +1,63 @@ +package application + +import ( + "context" + "log" + + "go.dev.pztrn.name/metricator/internal/storage" + "go.dev.pztrn.name/metricator/internal/storage/memory" +) + +// Application is a thing that responsible for all application-related +// actions like data fetching, storing, etc. on higher level. +type Application struct { + config *Config + ctx context.Context + doneChan chan struct{} + name string + + storage storage.Metrics + storageDone chan struct{} +} + +// NewApplication creates new application. +func NewApplication(ctx context.Context, name string, config *Config) *Application { + a := &Application{ + config: config, + ctx: ctx, + doneChan: make(chan struct{}), + name: name, + } + a.initialize() + + return a +} + +// GetDoneChan returns a channel which should be used to block execution until +// application's routines are completed. +func (a *Application) GetDoneChan() chan struct{} { + return a.doneChan +} + +// Initializes internal things like storage, HTTP client, etc. +func (a *Application) initialize() { + a.storage, a.storageDone = memory.NewStorage(a.ctx, a.name+" storage") + + log.Printf("Application '%s' initialized with configuration: %+v\n", a.name, a.config) +} + +// Start starts asynchronous things like data fetching, storage cleanup, etc. +func (a *Application) Start() { + a.storage.Start() + + // The Context Listening Goroutine. + go func() { + <-a.ctx.Done() + // We should wait until storage routines are also stopped. + <-a.storage.GetDoneChan() + + log.Println("Application", a.name, "stopped") + + a.doneChan <- struct{}{} + }() +} diff --git a/internal/application/config.go b/internal/application/config.go new file mode 100644 index 0000000..304182f --- /dev/null +++ b/internal/application/config.go @@ -0,0 +1,15 @@ +package application + +import "time" + +// Config is a generic application configuration. +type Config struct { + // Endpoint is a remote application endpoint which should give us metrics + // in Prometheus format. + Endpoint string + // Headers is a list of headers that should be added to metrics request. + Headers map[string]string + // TimeBetweenRequests is a minimal amount of time which should pass + // between requests. + TimeBetweenRequests time.Duration +} diff --git a/internal/application/fetcher.go b/internal/application/fetcher.go new file mode 100644 index 0000000..65574aa --- /dev/null +++ b/internal/application/fetcher.go @@ -0,0 +1,4 @@ +package application + +// Configures and starts Prometheus data fetching goroutine. +func (a *Application) startFetcher() {} diff --git a/internal/common/const.go b/internal/common/version_info.go similarity index 100% rename from internal/common/const.go rename to internal/common/version_info.go diff --git a/internal/configuration/config.go b/internal/configuration/config.go index 5239495..61a8f16 100644 --- a/internal/configuration/config.go +++ b/internal/configuration/config.go @@ -8,8 +8,8 @@ import ( "os" "path/filepath" "strings" - "time" + "go.dev.pztrn.name/metricator/internal/application" "gopkg.in/yaml.v2" ) @@ -23,23 +23,7 @@ type Config struct { configPath string // Applications describes configuration for remote application's endpoints. // Key is an application's name. - Applications map[string]struct { - // Endpoint is a remote application endpoint which should give us metrics - // in Prometheus format. - Endpoint string - // Headers is a list of headers that should be added to metrics request. - Headers map[string]string - // TimeBetweenRequests is a minimal amount of time which should pass - // between requests. - TimeBetweenRequests time.Duration - } - // Datastore describes data storage configuration. - Datastore struct { - // ValidTimeout is a timeout for which every data entry will be considered - // as valid. After that timeout if value wasn't updated it will be considered - // as invalid and purged from memory. - ValidTimeout time.Duration `yaml:"valid_timeout"` - } `yaml:"datastore"` + Applications map[string]*application.Config `yaml:"applications"` } // NewConfig returns new configuration. diff --git a/internal/datastore/application_store.go b/internal/datastore/application_store.go deleted file mode 100644 index 8ccc857..0000000 --- a/internal/datastore/application_store.go +++ /dev/null @@ -1,25 +0,0 @@ -package datastore - -import "sync" - -// This is application-specific data storage. -type applicationStorage struct { - metrics map[string]string - metricsMutex sync.RWMutex -} - -// Creates new application-specific storage. -func newApplicationStorage() *applicationStorage { - as := &applicationStorage{} - as.initialize() - - return as -} - -// Initializes internal things. -func (as *applicationStorage) initialize() { - as.metrics = make(map[string]string) -} - -// Starts application-specific things, like goroutine for HTTP requests. -func (as *applicationStorage) start() {} diff --git a/internal/datastore/datastore.go b/internal/datastore/datastore.go deleted file mode 100644 index c2bb65d..0000000 --- a/internal/datastore/datastore.go +++ /dev/null @@ -1,56 +0,0 @@ -package datastore - -import ( - "context" - "log" - "sync" - - "go.dev.pztrn.name/metricator/internal/configuration" -) - -// DataStore is a data storage structure. It keeps all gathered metrics and gives -// them away on request. -type DataStore struct { - config *configuration.Config - ctx context.Context - doneChan chan struct{} - - datas map[string]*applicationStorage - datasMutex sync.RWMutex -} - -// NewDataStore creates new data storage. -func NewDataStore(ctx context.Context, cfg *configuration.Config) (*DataStore, chan struct{}) { - ds := &DataStore{ - config: cfg, - ctx: ctx, - doneChan: make(chan struct{}), - } - ds.initialize() - - return ds, ds.doneChan -} - -// Internal things initialization. -func (ds *DataStore) initialize() { - ds.datas = make(map[string]*applicationStorage) - - // Create applications defined in configuration. - - go func() { - <-ds.ctx.Done() - log.Println("Data storage stopped") - - ds.doneChan <- struct{}{} - }() -} - -// Start starts data storage asynchronous things. -func (ds *DataStore) Start() { - log.Println("Starting data storage...") - - ds.datasMutex.RLock() - for _, storage := range ds.datas { - storage.start() - } -} diff --git a/internal/httpserver/httpserver.go b/internal/httpserver/httpserver.go index a3ce0e8..5009241 100644 --- a/internal/httpserver/httpserver.go +++ b/internal/httpserver/httpserver.go @@ -40,6 +40,8 @@ func (h *HTTPServer) getRequestContext(_ net.Listener) context.Context { // Initializes handler and HTTP server structure. func (h *HTTPServer) initialize() { h.handler = &handler{} + // We do not need to specify all possible parameters for HTTP server, so: + // nolint:exaustivestruct h.server = &http.Server{ // ToDo: make it all configurable. Addr: ":34421", diff --git a/internal/storage/generic.go b/internal/storage/generic.go new file mode 100644 index 0000000..d88a4d3 --- /dev/null +++ b/internal/storage/generic.go @@ -0,0 +1,15 @@ +package storage + +// GenericStorage describes interface every other storage should embed +// and conform to as it contains essential things like context handling. +type GenericStorage interface { + // Get returns data from storage by key. + Get(string) string + // GetDoneChan returns a channel which should be used to block execution + // until storage's routines are completed. + GetDoneChan() chan struct{} + // Put puts passed data into storage. + Put(map[string]string) + // Start starts asynchronous things if needed. + Start() +} diff --git a/internal/storage/memory/memory.go b/internal/storage/memory/memory.go new file mode 100644 index 0000000..cb30ce2 --- /dev/null +++ b/internal/storage/memory/memory.go @@ -0,0 +1,75 @@ +package memory + +import ( + "context" + "log" + "sync" +) + +// Storage is an in-memory storage. +type Storage struct { + ctx context.Context + doneChan chan struct{} + name string + + data map[string]string + dataMutex sync.RWMutex +} + +// NewStorage creates new in-memory storage to use. +func NewStorage(ctx context.Context, name string) (*Storage, chan struct{}) { + s := &Storage{ + ctx: ctx, + doneChan: make(chan struct{}), + name: name, + } + s.initialize() + + return s, s.doneChan +} + +// Get returns data from storage by key. +func (s *Storage) Get(key string) string { + s.dataMutex.RLock() + defer s.dataMutex.RUnlock() + + data, found := s.data[key] + if !found { + return "Not found" + } + + return data +} + +// GetDoneChan returns a channel which should be used to block execution +// until storage's routines are completed. +func (s *Storage) GetDoneChan() chan struct{} { + return s.doneChan +} + +// Initializes internal things. +func (s *Storage) initialize() { + s.data = make(map[string]string) +} + +// Put puts passed data into storage. +func (s *Storage) Put(data map[string]string) { + s.dataMutex.Lock() + defer s.dataMutex.Unlock() + + for k, v := range data { + s.data[k] = v + } +} + +// Start starts asynchronous things if needed. +func (s *Storage) Start() { + // The Context Listening Goroutine. + go func() { + <-s.ctx.Done() + + log.Println("In-memory storage", s.name, "done") + + s.doneChan <- struct{}{} + }() +} diff --git a/internal/storage/metrics.go b/internal/storage/metrics.go new file mode 100644 index 0000000..914e7f3 --- /dev/null +++ b/internal/storage/metrics.go @@ -0,0 +1,6 @@ +package storage + +// Metrics describes generic metrics storage interface. +type Metrics interface { + GenericStorage +} diff --git a/metricator.example.yaml b/metricator.example.yaml index 824c903..36c2c78 100644 --- a/metricator.example.yaml +++ b/metricator.example.yaml @@ -1,2 +1,12 @@ -datastore: - valid_timeout: 30s +# Applications configuration. +applications: + # Application name. + theveryexampleapp: + # Endpoint to fetch. HTTP GET will be used. + endpoint: http://127.0.0.1:8789/metrics + # Headers to append to request. + headers: + X-API-KEY: th4apik3yh4rdt0gu3ss + # Timeout between requests. Not neccessarily be exact and requests might + # be sent in 60 or more seconds (in this example). + time_between_requests: 60s