Rethink how metricator will work, golangci-lint and gitlab ci configs.

This commit is contained in:
Stanislav Nikitin 2020-11-29 03:22:39 +05:00
parent e679fecf6e
commit f915470c0b
Signed by: pztrn
GPG Key ID: 1E944A0F0568B550
20 changed files with 256 additions and 112 deletions

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
._bin ._bin
.vscode .vscode
metricator.yaml

10
.gitlab-ci.yml Normal file
View File

@ -0,0 +1,10 @@
stages:
- test
lint:
stage: test
tags:
- docker
image: golangci/golangci-lint:v1.32.2
script:
- golangci-lint run

23
.golangci.yml Normal file
View File

@ -0,0 +1,23 @@
run:
deadline: 5m
linters:
enable-all: true
disable:
# Because globals might exist, but according to our codestyle they
# should be lowercased and considered as unexported.
- gochecknoglobals
# While it might be useful it'll create more problems that will solve.
- gocritic
# Complains about main() lengths, which isn't an issue.
- funlen
# Magic numbers might be everywhere. Disabled for now.
- gomnd
# ToDos everywhere
- godox
# Why? WHY? WHY _test???
- testpackage
linters-settings:
lll:
line-length: 120
gocyclo:
min-complexity: 40

View File

@ -1,6 +1,8 @@
# This is a Metricator Makefile. # This is a Metricator Makefile.
# It contains calls to scripts placed in scripts directory. # It contains calls to scripts placed in scripts directory.
CONFIG ?= ./metricator.example.yaml
help: Makefile help: Makefile
@echo -e "Metricator Makefile available subcommands:\n" @echo -e "Metricator Makefile available subcommands:\n"
@cat $< | grep "## " | sort | sed -n 's/^## //p' @cat $< | grep "## " | sort | sed -n 's/^## //p'
@ -19,7 +21,7 @@ metricatord-build: check-build-dir
## metricatord-run: starts metricator daemon. ## metricatord-run: starts metricator daemon.
metricatord-run: metricatord-build metricatord-run: metricatord-build
./._bin/metricatord -config ./metricator.example.yaml ./._bin/metricatord -config ${CONFIG}
show-git-data: show-git-data:
@echo "Parameters for current source code state:" @echo "Parameters for current source code state:"

View File

@ -14,12 +14,11 @@ Metricator will issue only one request and cache data in memory between them. Al
## Installation ## Installation
*TBW* TBW
## Configuration ## Configuration
*TBW* TBW
## Documentation ## Documentation

View File

@ -8,9 +8,9 @@ import (
"os/signal" "os/signal"
"syscall" "syscall"
"go.dev.pztrn.name/metricator/internal/application"
"go.dev.pztrn.name/metricator/internal/common" "go.dev.pztrn.name/metricator/internal/common"
"go.dev.pztrn.name/metricator/internal/configuration" "go.dev.pztrn.name/metricator/internal/configuration"
"go.dev.pztrn.name/metricator/internal/datastore"
"go.dev.pztrn.name/metricator/internal/httpserver" "go.dev.pztrn.name/metricator/internal/httpserver"
) )
@ -26,16 +26,27 @@ func main() {
config := configuration.NewConfig() config := configuration.NewConfig()
httpSrv, httpStopped := httpserver.NewHTTPServer(mainCtx, config) httpSrv, httpStopped := httpserver.NewHTTPServer(mainCtx, config)
dataStore, dataStoreStopped := datastore.NewDataStore(mainCtx, config)
// Parse configuration.
flag.Parse() flag.Parse()
err := config.Parse() err := config.Parse()
if err != nil { if err != nil {
log.Fatalln("Failed to parse configuration:", err.Error()) log.Fatalln("Failed to parse configuration:", err.Error())
} }
log.Printf("Configuration parsed: %+v\n", config) log.Printf("Configuration parsed: %+v\n", config)
dataStore.Start() // Create applications.
apps := make([]*application.Application, 0, len(config.Applications))
for appName, appConfig := range config.Applications {
app := application.NewApplication(mainCtx, appName, appConfig)
app.Start()
apps = append(apps, app)
}
httpSrv.Start() httpSrv.Start()
log.Println("Metricator is started and ready to serve requests") log.Println("Metricator is started and ready to serve requests")
@ -46,15 +57,18 @@ func main() {
signal.Notify(signalHandler, os.Interrupt, syscall.SIGTERM) signal.Notify(signalHandler, os.Interrupt, syscall.SIGTERM)
go func() { go func(apps []*application.Application) {
<-signalHandler <-signalHandler
cancelFunc() cancelFunc()
<-dataStoreStopped for _, app := range apps {
<-app.GetDoneChan()
}
<-httpStopped <-httpStopped
shutdownDone <- true shutdownDone <- true
}() }(apps)
<-shutdownDone <-shutdownDone
log.Println("Metricator stopped") log.Println("Metricator stopped")

2
go.mod
View File

@ -2,4 +2,4 @@ module go.dev.pztrn.name/metricator
go 1.13 go 1.13
require gopkg.in/yaml.v2 v2.3.0 require gopkg.in/yaml.v2 v2.4.0

2
go.sum
View File

@ -1,3 +1,5 @@
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

View File

@ -0,0 +1,63 @@
package application
import (
"context"
"log"
"go.dev.pztrn.name/metricator/internal/storage"
"go.dev.pztrn.name/metricator/internal/storage/memory"
)
// Application is a thing that responsible for all application-related
// actions like data fetching, storing, etc. on higher level.
type Application struct {
config *Config
ctx context.Context
doneChan chan struct{}
name string
storage storage.Metrics
storageDone chan struct{}
}
// NewApplication creates new application.
func NewApplication(ctx context.Context, name string, config *Config) *Application {
a := &Application{
config: config,
ctx: ctx,
doneChan: make(chan struct{}),
name: name,
}
a.initialize()
return a
}
// GetDoneChan returns a channel which should be used to block execution until
// application's routines are completed.
func (a *Application) GetDoneChan() chan struct{} {
return a.doneChan
}
// Initializes internal things like storage, HTTP client, etc.
func (a *Application) initialize() {
a.storage, a.storageDone = memory.NewStorage(a.ctx, a.name+" storage")
log.Printf("Application '%s' initialized with configuration: %+v\n", a.name, a.config)
}
// Start starts asynchronous things like data fetching, storage cleanup, etc.
func (a *Application) Start() {
a.storage.Start()
// The Context Listening Goroutine.
go func() {
<-a.ctx.Done()
// We should wait until storage routines are also stopped.
<-a.storage.GetDoneChan()
log.Println("Application", a.name, "stopped")
a.doneChan <- struct{}{}
}()
}

View File

@ -0,0 +1,15 @@
package application
import "time"
// Config is a generic application configuration.
type Config struct {
// Endpoint is a remote application endpoint which should give us metrics
// in Prometheus format.
Endpoint string
// Headers is a list of headers that should be added to metrics request.
Headers map[string]string
// TimeBetweenRequests is a minimal amount of time which should pass
// between requests.
TimeBetweenRequests time.Duration
}

View File

@ -0,0 +1,4 @@
package application
// Configures and starts Prometheus data fetching goroutine.
func (a *Application) startFetcher() {}

View File

@ -8,8 +8,8 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"time"
"go.dev.pztrn.name/metricator/internal/application"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
) )
@ -23,23 +23,7 @@ type Config struct {
configPath string configPath string
// Applications describes configuration for remote application's endpoints. // Applications describes configuration for remote application's endpoints.
// Key is an application's name. // Key is an application's name.
Applications map[string]struct { Applications map[string]*application.Config `yaml:"applications"`
// Endpoint is a remote application endpoint which should give us metrics
// in Prometheus format.
Endpoint string
// Headers is a list of headers that should be added to metrics request.
Headers map[string]string
// TimeBetweenRequests is a minimal amount of time which should pass
// between requests.
TimeBetweenRequests time.Duration
}
// Datastore describes data storage configuration.
Datastore struct {
// ValidTimeout is a timeout for which every data entry will be considered
// as valid. After that timeout if value wasn't updated it will be considered
// as invalid and purged from memory.
ValidTimeout time.Duration `yaml:"valid_timeout"`
} `yaml:"datastore"`
} }
// NewConfig returns new configuration. // NewConfig returns new configuration.

View File

@ -1,25 +0,0 @@
package datastore
import "sync"
// This is application-specific data storage.
type applicationStorage struct {
metrics map[string]string
metricsMutex sync.RWMutex
}
// Creates new application-specific storage.
func newApplicationStorage() *applicationStorage {
as := &applicationStorage{}
as.initialize()
return as
}
// Initializes internal things.
func (as *applicationStorage) initialize() {
as.metrics = make(map[string]string)
}
// Starts application-specific things, like goroutine for HTTP requests.
func (as *applicationStorage) start() {}

View File

@ -1,56 +0,0 @@
package datastore
import (
"context"
"log"
"sync"
"go.dev.pztrn.name/metricator/internal/configuration"
)
// DataStore is a data storage structure. It keeps all gathered metrics and gives
// them away on request.
type DataStore struct {
config *configuration.Config
ctx context.Context
doneChan chan struct{}
datas map[string]*applicationStorage
datasMutex sync.RWMutex
}
// NewDataStore creates new data storage.
func NewDataStore(ctx context.Context, cfg *configuration.Config) (*DataStore, chan struct{}) {
ds := &DataStore{
config: cfg,
ctx: ctx,
doneChan: make(chan struct{}),
}
ds.initialize()
return ds, ds.doneChan
}
// Internal things initialization.
func (ds *DataStore) initialize() {
ds.datas = make(map[string]*applicationStorage)
// Create applications defined in configuration.
go func() {
<-ds.ctx.Done()
log.Println("Data storage stopped")
ds.doneChan <- struct{}{}
}()
}
// Start starts data storage asynchronous things.
func (ds *DataStore) Start() {
log.Println("Starting data storage...")
ds.datasMutex.RLock()
for _, storage := range ds.datas {
storage.start()
}
}

View File

@ -40,6 +40,8 @@ func (h *HTTPServer) getRequestContext(_ net.Listener) context.Context {
// Initializes handler and HTTP server structure. // Initializes handler and HTTP server structure.
func (h *HTTPServer) initialize() { func (h *HTTPServer) initialize() {
h.handler = &handler{} h.handler = &handler{}
// We do not need to specify all possible parameters for HTTP server, so:
// nolint:exaustivestruct
h.server = &http.Server{ h.server = &http.Server{
// ToDo: make it all configurable. // ToDo: make it all configurable.
Addr: ":34421", Addr: ":34421",

View File

@ -0,0 +1,15 @@
package storage
// GenericStorage describes interface every other storage should embed
// and conform to as it contains essential things like context handling.
type GenericStorage interface {
// Get returns data from storage by key.
Get(string) string
// GetDoneChan returns a channel which should be used to block execution
// until storage's routines are completed.
GetDoneChan() chan struct{}
// Put puts passed data into storage.
Put(map[string]string)
// Start starts asynchronous things if needed.
Start()
}

View File

@ -0,0 +1,75 @@
package memory
import (
"context"
"log"
"sync"
)
// Storage is an in-memory storage.
type Storage struct {
ctx context.Context
doneChan chan struct{}
name string
data map[string]string
dataMutex sync.RWMutex
}
// NewStorage creates new in-memory storage to use.
func NewStorage(ctx context.Context, name string) (*Storage, chan struct{}) {
s := &Storage{
ctx: ctx,
doneChan: make(chan struct{}),
name: name,
}
s.initialize()
return s, s.doneChan
}
// Get returns data from storage by key.
func (s *Storage) Get(key string) string {
s.dataMutex.RLock()
defer s.dataMutex.RUnlock()
data, found := s.data[key]
if !found {
return "Not found"
}
return data
}
// GetDoneChan returns a channel which should be used to block execution
// until storage's routines are completed.
func (s *Storage) GetDoneChan() chan struct{} {
return s.doneChan
}
// Initializes internal things.
func (s *Storage) initialize() {
s.data = make(map[string]string)
}
// Put puts passed data into storage.
func (s *Storage) Put(data map[string]string) {
s.dataMutex.Lock()
defer s.dataMutex.Unlock()
for k, v := range data {
s.data[k] = v
}
}
// Start starts asynchronous things if needed.
func (s *Storage) Start() {
// The Context Listening Goroutine.
go func() {
<-s.ctx.Done()
log.Println("In-memory storage", s.name, "done")
s.doneChan <- struct{}{}
}()
}

View File

@ -0,0 +1,6 @@
package storage
// Metrics describes generic metrics storage interface.
type Metrics interface {
GenericStorage
}

View File

@ -1,2 +1,12 @@
datastore: # Applications configuration.
valid_timeout: 30s applications:
# Application name.
theveryexampleapp:
# Endpoint to fetch. HTTP GET will be used.
endpoint: http://127.0.0.1:8789/metrics
# Headers to append to request.
headers:
X-API-KEY: th4apik3yh4rdt0gu3ss
# Timeout between requests. Not neccessarily be exact and requests might
# be sent in 60 or more seconds (in this example).
time_between_requests: 60s