Rethink how metricator will work, golangci-lint and gitlab ci configs.

.gitignore

@@ -1,2 +1,3 @@
 ._bin
 .vscode
+metricator.yaml

.gitlab-ci.yml

@@ -0,0 +1,10 @@
+stages:
+  - test
+
+lint:
+  stage: test
+  tags:
+    - docker
+  image: golangci/golangci-lint:v1.32.2
+  script:
+    - golangci-lint run

.golangci.yml

@@ -0,0 +1,23 @@
+run:
+  deadline: 5m
+linters:
+  enable-all: true
+  disable:
+    # Because globals might exist, but according to our codestyle they
+    # should be lowercased and considered as unexported.
+    - gochecknoglobals
+    # While it might be useful it'll create more problems that will solve.
+    - gocritic
+    # Complains about main() lengths, which isn't an issue.
+    - funlen
+    # Magic numbers might be everywhere. Disabled for now.
+    - gomnd
+    # ToDos everywhere
+    - godox
+    # Why? WHY? WHY _test???
+    - testpackage
+linters-settings:
+  lll:
+    line-length: 120
+  gocyclo:
+    min-complexity: 40

Makefile

@@ -1,6 +1,8 @@
 # This is a Metricator Makefile.
 # It contains calls to scripts placed in scripts directory.
 
+CONFIG ?= ./metricator.example.yaml
+
 help: Makefile
 	@echo -e "Metricator Makefile available subcommands:\n"
 	@cat $< | grep "## " | sort | sed -n 's/^## //p'
@@ -19,7 +21,7 @@ metricatord-build: check-build-dir
 
 ## metricatord-run: starts metricator daemon.
 metricatord-run: metricatord-build
-	./._bin/metricatord -config ./metricator.example.yaml
+	./._bin/metricatord -config ${CONFIG}
 
 show-git-data:
 	@echo "Parameters for current source code state:"

README.md

@@ -14,12 +14,11 @@ Metricator will issue only one request and cache data in memory between them. Al
 
 ## Installation
 
-*TBW*
+TBW
 
 ## Configuration
 
-*TBW*
-
+TBW
 
 ## Documentation
 

cmd/metricatord/main.go

@@ -8,9 +8,9 @@ import (
 	"os/signal"
 	"syscall"
 
+	"go.dev.pztrn.name/metricator/internal/application"
 	"go.dev.pztrn.name/metricator/internal/common"
 	"go.dev.pztrn.name/metricator/internal/configuration"
-	"go.dev.pztrn.name/metricator/internal/datastore"
 	"go.dev.pztrn.name/metricator/internal/httpserver"
 )
 
@@ -26,16 +26,27 @@ func main() {
 	config := configuration.NewConfig()
 
 	httpSrv, httpStopped := httpserver.NewHTTPServer(mainCtx, config)
-	dataStore, dataStoreStopped := datastore.NewDataStore(mainCtx, config)
 
+	// Parse configuration.
 	flag.Parse()
+
 	err := config.Parse()
 	if err != nil {
 		log.Fatalln("Failed to parse configuration:", err.Error())
 	}
+
 	log.Printf("Configuration parsed: %+v\n", config)
 
-	dataStore.Start()
+	// Create applications.
+	apps := make([]*application.Application, 0, len(config.Applications))
+
+	for appName, appConfig := range config.Applications {
+		app := application.NewApplication(mainCtx, appName, appConfig)
+		app.Start()
+
+		apps = append(apps, app)
+	}
+
 	httpSrv.Start()
 
 	log.Println("Metricator is started and ready to serve requests")
@@ -46,15 +57,18 @@ func main() {
 
 	signal.Notify(signalHandler, os.Interrupt, syscall.SIGTERM)
 
-	go func() {
+	go func(apps []*application.Application) {
 		<-signalHandler
 		cancelFunc()
 
-		<-dataStoreStopped
+		for _, app := range apps {
+			<-app.GetDoneChan()
+		}
+
 		<-httpStopped
 
 		shutdownDone <- true
-	}()
+	}(apps)
 
 	<-shutdownDone
 	log.Println("Metricator stopped")

go.mod

@@ -2,4 +2,4 @@ module go.dev.pztrn.name/metricator
 
 go 1.13
 
-require gopkg.in/yaml.v2 v2.3.0
+require gopkg.in/yaml.v2 v2.4.0

go.sum

@@ -1,3 +1,5 @@
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

internal/application/application.go

@@ -0,0 +1,63 @@
+package application
+
+import (
+	"context"
+	"log"
+
+	"go.dev.pztrn.name/metricator/internal/storage"
+	"go.dev.pztrn.name/metricator/internal/storage/memory"
+)
+
+// Application is a thing that responsible for all application-related
+// actions like data fetching, storing, etc. on higher level.
+type Application struct {
+	config   *Config
+	ctx      context.Context
+	doneChan chan struct{}
+	name     string
+
+	storage     storage.Metrics
+	storageDone chan struct{}
+}
+
+// NewApplication creates new application.
+func NewApplication(ctx context.Context, name string, config *Config) *Application {
+	a := &Application{
+		config:   config,
+		ctx:      ctx,
+		doneChan: make(chan struct{}),
+		name:     name,
+	}
+	a.initialize()
+
+	return a
+}
+
+// GetDoneChan returns a channel which should be used to block execution until
+// application's routines are completed.
+func (a *Application) GetDoneChan() chan struct{} {
+	return a.doneChan
+}
+
+// Initializes internal things like storage, HTTP client, etc.
+func (a *Application) initialize() {
+	a.storage, a.storageDone = memory.NewStorage(a.ctx, a.name+" storage")
+
+	log.Printf("Application '%s' initialized with configuration: %+v\n", a.name, a.config)
+}
+
+// Start starts asynchronous things like data fetching, storage cleanup, etc.
+func (a *Application) Start() {
+	a.storage.Start()
+
+	// The Context Listening Goroutine.
+	go func() {
+		<-a.ctx.Done()
+		// We should wait until storage routines are also stopped.
+		<-a.storage.GetDoneChan()
+
+		log.Println("Application", a.name, "stopped")
+
+		a.doneChan <- struct{}{}
+	}()
+}

internal/application/config.go

@@ -0,0 +1,15 @@
+package application
+
+import "time"
+
+// Config is a generic application configuration.
+type Config struct {
+	// Endpoint is a remote application endpoint which should give us metrics
+	// in Prometheus format.
+	Endpoint string
+	// Headers is a list of headers that should be added to metrics request.
+	Headers map[string]string
+	// TimeBetweenRequests is a minimal amount of time which should pass
+	// between requests.
+	TimeBetweenRequests time.Duration
+}

internal/application/fetcher.go

@@ -0,0 +1,4 @@
+package application
+
+// Configures and starts Prometheus data fetching goroutine.
+func (a *Application) startFetcher() {}

internal/configuration/config.go

@@ -8,8 +8,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
+	"go.dev.pztrn.name/metricator/internal/application"
 	"gopkg.in/yaml.v2"
 )
 
@@ -23,23 +23,7 @@ type Config struct {
 	configPath string
 	// Applications describes configuration for remote application's endpoints.
 	// Key is an application's name.
-	Applications map[string]struct {
-		// Endpoint is a remote application endpoint which should give us metrics
-		// in Prometheus format.
-		Endpoint string
-		// Headers is a list of headers that should be added to metrics request.
-		Headers map[string]string
-		// TimeBetweenRequests is a minimal amount of time which should pass
-		// between requests.
-		TimeBetweenRequests time.Duration
-	}
-	// Datastore describes data storage configuration.
-	Datastore struct {
-		// ValidTimeout is a timeout for which every data entry will be considered
-		// as valid. After that timeout if value wasn't updated it will be considered
-		// as invalid and purged from memory.
-		ValidTimeout time.Duration `yaml:"valid_timeout"`
-	} `yaml:"datastore"`
+	Applications map[string]*application.Config `yaml:"applications"`
 }
 
 // NewConfig returns new configuration.

internal/datastore/application_store.go

@@ -1,25 +0,0 @@
-package datastore
-
-import "sync"
-
-// This is application-specific data storage.
-type applicationStorage struct {
-	metrics      map[string]string
-	metricsMutex sync.RWMutex
-}
-
-// Creates new application-specific storage.
-func newApplicationStorage() *applicationStorage {
-	as := &applicationStorage{}
-	as.initialize()
-
-	return as
-}
-
-// Initializes internal things.
-func (as *applicationStorage) initialize() {
-	as.metrics = make(map[string]string)
-}
-
-// Starts application-specific things, like goroutine for HTTP requests.
-func (as *applicationStorage) start() {}

internal/datastore/datastore.go

@@ -1,56 +0,0 @@
-package datastore
-
-import (
-	"context"
-	"log"
-	"sync"
-
-	"go.dev.pztrn.name/metricator/internal/configuration"
-)
-
-// DataStore is a data storage structure. It keeps all gathered metrics and gives
-//  them away on request.
-type DataStore struct {
-	config   *configuration.Config
-	ctx      context.Context
-	doneChan chan struct{}
-
-	datas      map[string]*applicationStorage
-	datasMutex sync.RWMutex
-}
-
-// NewDataStore creates new data storage.
-func NewDataStore(ctx context.Context, cfg *configuration.Config) (*DataStore, chan struct{}) {
-	ds := &DataStore{
-		config:   cfg,
-		ctx:      ctx,
-		doneChan: make(chan struct{}),
-	}
-	ds.initialize()
-
-	return ds, ds.doneChan
-}
-
-// Internal things initialization.
-func (ds *DataStore) initialize() {
-	ds.datas = make(map[string]*applicationStorage)
-
-	// Create applications defined in configuration.
-
-	go func() {
-		<-ds.ctx.Done()
-		log.Println("Data storage stopped")
-
-		ds.doneChan <- struct{}{}
-	}()
-}
-
-// Start starts data storage asynchronous things.
-func (ds *DataStore) Start() {
-	log.Println("Starting data storage...")
-
-	ds.datasMutex.RLock()
-	for _, storage := range ds.datas {
-		storage.start()
-	}
-}

internal/httpserver/httpserver.go

@@ -40,6 +40,8 @@ func (h *HTTPServer) getRequestContext(_ net.Listener) context.Context {
 // Initializes handler and HTTP server structure.
 func (h *HTTPServer) initialize() {
 	h.handler = &handler{}
+	// We do not need to specify all possible parameters for HTTP server, so:
+	// nolint:exaustivestruct
 	h.server = &http.Server{
 		// ToDo: make it all configurable.
 		Addr:           ":34421",

internal/storage/generic.go

@@ -0,0 +1,15 @@
+package storage
+
+// GenericStorage describes interface every other storage should embed
+// and conform to as it contains essential things like context handling.
+type GenericStorage interface {
+	// Get returns data from storage by key.
+	Get(string) string
+	// GetDoneChan returns a channel which should be used to block execution
+	// until storage's routines are completed.
+	GetDoneChan() chan struct{}
+	// Put puts passed data into storage.
+	Put(map[string]string)
+	// Start starts asynchronous things if needed.
+	Start()
+}

internal/storage/memory/memory.go

@@ -0,0 +1,75 @@
+package memory
+
+import (
+	"context"
+	"log"
+	"sync"
+)
+
+// Storage is an in-memory storage.
+type Storage struct {
+	ctx      context.Context
+	doneChan chan struct{}
+	name     string
+
+	data      map[string]string
+	dataMutex sync.RWMutex
+}
+
+// NewStorage creates new in-memory storage to use.
+func NewStorage(ctx context.Context, name string) (*Storage, chan struct{}) {
+	s := &Storage{
+		ctx:      ctx,
+		doneChan: make(chan struct{}),
+		name:     name,
+	}
+	s.initialize()
+
+	return s, s.doneChan
+}
+
+// Get returns data from storage by key.
+func (s *Storage) Get(key string) string {
+	s.dataMutex.RLock()
+	defer s.dataMutex.RUnlock()
+
+	data, found := s.data[key]
+	if !found {
+		return "Not found"
+	}
+
+	return data
+}
+
+// GetDoneChan returns a channel which should be used to block execution
+// until storage's routines are completed.
+func (s *Storage) GetDoneChan() chan struct{} {
+	return s.doneChan
+}
+
+// Initializes internal things.
+func (s *Storage) initialize() {
+	s.data = make(map[string]string)
+}
+
+// Put puts passed data into storage.
+func (s *Storage) Put(data map[string]string) {
+	s.dataMutex.Lock()
+	defer s.dataMutex.Unlock()
+
+	for k, v := range data {
+		s.data[k] = v
+	}
+}
+
+// Start starts asynchronous things if needed.
+func (s *Storage) Start() {
+	// The Context Listening Goroutine.
+	go func() {
+		<-s.ctx.Done()
+
+		log.Println("In-memory storage", s.name, "done")
+
+		s.doneChan <- struct{}{}
+	}()
+}

internal/storage/metrics.go

@@ -0,0 +1,6 @@
+package storage
+
+// Metrics describes generic metrics storage interface.
+type Metrics interface {
+	GenericStorage
+}

metricator.example.yaml

@@ -1,2 +1,12 @@
-datastore:
-  valid_timeout: 30s
+# Applications configuration.
+applications:
+  # Application name.
+  theveryexampleapp:
+    # Endpoint to fetch. HTTP GET will be used.
+    endpoint: http://127.0.0.1:8789/metrics
+    # Headers to append to request.
+    headers:
+      X-API-KEY: th4apik3yh4rdt0gu3ss
+    # Timeout between requests. Not neccessarily be exact and requests might
+    # be sent in 60 or more seconds (in this example).
+    time_between_requests: 60s