From cdf9997cfe7e9c03b2cc891a8d625ab1761b4fb5 Mon Sep 17 00:00:00 2001 From: "Stanislav N. aka pztrn" Date: Sat, 28 Nov 2020 23:34:20 +0500 Subject: [PATCH] Initial commit. --- .gitignore | 2 + Makefile | 26 +++++++ README.md | 26 +++++++ cmd/metricatord/main.go | 62 +++++++++++++++++ docs/INDEX.md | 13 ++++ go.mod | 5 ++ go.sum | 3 + internal/common/const.go | 8 +++ internal/common/context_keys.go | 11 +++ internal/common/http_handler.go | 6 ++ internal/configuration/config.go | 93 +++++++++++++++++++++++++ internal/datastore/application_store.go | 25 +++++++ internal/datastore/datastore.go | 56 +++++++++++++++ internal/httpserver/handler.go | 20 ++++++ internal/httpserver/httpserver.go | 79 +++++++++++++++++++++ metricator.example.yaml | 2 + scripts/README.md | 17 +++++ scripts/build.sh | 5 ++ scripts/shell_helpers/get_git_data.sh | 7 ++ scripts/show_git_data.sh | 9 +++ 20 files changed, 475 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 cmd/metricatord/main.go create mode 100644 docs/INDEX.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/common/const.go create mode 100644 internal/common/context_keys.go create mode 100644 internal/common/http_handler.go create mode 100644 internal/configuration/config.go create mode 100644 internal/datastore/application_store.go create mode 100644 internal/datastore/datastore.go create mode 100644 internal/httpserver/handler.go create mode 100644 internal/httpserver/httpserver.go create mode 100644 metricator.example.yaml create mode 100644 scripts/README.md create mode 100755 scripts/build.sh create mode 100644 scripts/shell_helpers/get_git_data.sh create mode 100755 scripts/show_git_data.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6e17a58 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +._bin +.vscode diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1042eb8 --- /dev/null +++ b/Makefile @@ -0,0 +1,26 @@ +# This is a Metricator Makefile. +# It contains calls to scripts placed in scripts directory. + +help: Makefile + @echo -e "Metricator Makefile available subcommands:\n" + @cat $< | grep "## " | sort | sed -n 's/^## //p' + @echo "" + @make -f Makefile show-git-data + +.DEFAULT_GOAL := help + +check-build-dir: + @if [ ! -d "._bin" ]; then mkdir ._bin; fi + +## metricatord-build: builds metricator daemon and places into ${PWD}/._bin. +metricatord-build: check-build-dir + rm ./._bin/metricatord || true + cd cmd/metricatord && go build -o ../../._bin/metricatord + +## metricatord-run: starts metricator daemon. +metricatord-run: metricatord-build + ./._bin/metricatord -config ./metricator.example.yaml + +show-git-data: + @echo "Parameters for current source code state:" + @scripts/show_git_data.sh diff --git a/README.md b/README.md new file mode 100644 index 0000000..479c7f0 --- /dev/null +++ b/README.md @@ -0,0 +1,26 @@ +# Metricator + +Simple proxy between prometheus-powered application and your NMS. + +## Why + +I'm using NetXMS to monitor all of my systems. Many things I use exports metrics in prometheus format which can be utilized by custom parsing script. But I've encounter a performance problem when I need to monitor 30 metrics - parsing script will make 30 requests to prometheus endpoint which might affect performance. + +Metricator will issue only one request and cache data in memory between them. Also it will expose HTTP API to get single metric which can be easily utilized with any NMS. + +## Caveats + +* No authorization. **DO NOT** expose Metricator to wild world! + +## Installation + +*TBW* + +## Configuration + +*TBW* + + +## Documentation + +Check [docs directory](/docs/INDEX.md). diff --git a/cmd/metricatord/main.go b/cmd/metricatord/main.go new file mode 100644 index 0000000..3a53d33 --- /dev/null +++ b/cmd/metricatord/main.go @@ -0,0 +1,62 @@ +package main + +import ( + "context" + "flag" + "log" + "os" + "os/signal" + "syscall" + + "go.dev.pztrn.name/metricator/internal/common" + "go.dev.pztrn.name/metricator/internal/configuration" + "go.dev.pztrn.name/metricator/internal/datastore" + "go.dev.pztrn.name/metricator/internal/httpserver" +) + +func main() { + log.Printf("Starting Metricator daemon, version %s from branch %s (build #%s, commit hash %s)\n", + common.Version, + common.Branch, + common.Build, + common.CommitHash, + ) + + mainCtx, cancelFunc := context.WithCancel(context.Background()) + config := configuration.NewConfig() + + httpSrv, httpStopped := httpserver.NewHTTPServer(mainCtx, config) + dataStore, dataStoreStopped := datastore.NewDataStore(mainCtx, config) + + flag.Parse() + err := config.Parse() + if err != nil { + log.Fatalln("Failed to parse configuration:", err.Error()) + } + log.Printf("Configuration parsed: %+v\n", config) + + dataStore.Start() + httpSrv.Start() + + log.Println("Metricator is started and ready to serve requests") + + // CTRL+C handler. + signalHandler := make(chan os.Signal, 1) + shutdownDone := make(chan bool, 1) + + signal.Notify(signalHandler, os.Interrupt, syscall.SIGTERM) + + go func() { + <-signalHandler + cancelFunc() + + <-dataStoreStopped + <-httpStopped + + shutdownDone <- true + }() + + <-shutdownDone + log.Println("Metricator stopped") + os.Exit(0) +} diff --git a/docs/INDEX.md b/docs/INDEX.md new file mode 100644 index 0000000..7d8be9a --- /dev/null +++ b/docs/INDEX.md @@ -0,0 +1,13 @@ +# Metricator + +Welcome to Metricator docs. + +## What's the idea + +Main idea for Metricator is to provide high-speed proxy between classic NMSes (like NetXMS, Nagios and so on) and other software that exposes metrics in Prometheus format. + +Why proxy? Let's go by example. + +Imagine that software you wish to monitor with classic NMS exposes 250 metric items and you wrote a simple script which returns needed data. Classic NMSes is able to process only one value per checker (usually), which means that processing every metric will do 250 HTTP requests to monitored software which is obviosly not good. + +Metricator instead acts like Prometheus itself from monitored software PoV: it performs one HTTP request each timeout (configured in config), parses data and making it available to other requesters. As parsed metric data stored in memory it will be blazing fast and won't overload your system. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7f39460 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module go.dev.pztrn.name/metricator + +go 1.13 + +require gopkg.in/yaml.v2 v2.3.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8fabe8d --- /dev/null +++ b/go.sum @@ -0,0 +1,3 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/common/const.go b/internal/common/const.go new file mode 100644 index 0000000..92df57c --- /dev/null +++ b/internal/common/const.go @@ -0,0 +1,8 @@ +package common + +var ( + Branch string + Build string + CommitHash string + Version string +) diff --git a/internal/common/context_keys.go b/internal/common/context_keys.go new file mode 100644 index 0000000..8a4d759 --- /dev/null +++ b/internal/common/context_keys.go @@ -0,0 +1,11 @@ +package common + +// ContextKey is a type of context.Context keys. +type ContextKey string + +const ( + // ContextKeyApplication specifies that returned value is a name of application. + ContextKeyApplication ContextKey = "applicationName" + // ContextKeyMetric specifies that returned value is a name of metric of application. + ContextKeyMetric ContextKey = "metricName" +) diff --git a/internal/common/http_handler.go b/internal/common/http_handler.go new file mode 100644 index 0000000..09808d2 --- /dev/null +++ b/internal/common/http_handler.go @@ -0,0 +1,6 @@ +package common + +import "context" + +// HTTPHandlerFunc describes signature of HTTP requests handling function. +type HTTPHandlerFunc func(context.Context) string diff --git a/internal/configuration/config.go b/internal/configuration/config.go new file mode 100644 index 0000000..56b9dc5 --- /dev/null +++ b/internal/configuration/config.go @@ -0,0 +1,93 @@ +package configuration + +import ( + "errors" + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "gopkg.in/yaml.v2" +) + +var ( + ErrConfigurationFileDoesNotExist = errors.New("configuration file does not exist") + ErrConfigurationFilePathUndefined = errors.New("configuration file path wasn't provided") +) + +// Config is an application's configuration. +type Config struct { + configPath string + // Applications describes configuration for remote application's endpoints. + // Key is an application's name. + Applications map[string]struct { + // Endpoint is a remote application endpoint which should give us metrics + // in Prometheus format. + Endpoint string + // TimeBetweenRequests is a minimal amount of time which should pass + // between requests. + TimeBetweenRequests time.Duration + } + // Datastore describes data storage configuration. + Datastore struct { + // ValidTimeout is a timeout for which every data entry will be considered + // as valid. After that timeout if value wasn't updated it will be considered + // as invalid and purged from memory. + ValidTimeout time.Duration `yaml:"valid_timeout"` + } `yaml:"datastore"` +} + +// NewConfig returns new configuration. +func NewConfig() *Config { + c := &Config{} + c.initialize() + + return c +} + +// Initializes configuration. +func (c *Config) initialize() { + flag.StringVar(&c.configPath, "config", "", "Configuration file path.") +} + +// Parse parses configuration. +func (c *Config) Parse() error { + if c.configPath == "" { + return ErrConfigurationFilePathUndefined + } + + if strings.HasPrefix(c.configPath, "~") { + userDir, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("%s: %w", "file path normalization: getting user's home directory", err) + } + + c.configPath = strings.Replace(c.configPath, "~", userDir, 1) + } + + cfgPath, err := filepath.Abs(c.configPath) + if err != nil { + return fmt.Errorf("%s: %w", "file path normalization: getting absolute path", err) + } + + c.configPath = cfgPath + + if c.configPath == "" { + return fmt.Errorf("%s: %w", "file path normalization", ErrConfigurationFilePathUndefined) + } + + fileData, err := ioutil.ReadFile(c.configPath) + if err != nil { + return fmt.Errorf("%s: %w", "configuration file read", err) + } + + err = yaml.Unmarshal(fileData, c) + if err != nil { + return fmt.Errorf("%s: %w", "configuration file parsing", err) + } + + return nil +} diff --git a/internal/datastore/application_store.go b/internal/datastore/application_store.go new file mode 100644 index 0000000..8ccc857 --- /dev/null +++ b/internal/datastore/application_store.go @@ -0,0 +1,25 @@ +package datastore + +import "sync" + +// This is application-specific data storage. +type applicationStorage struct { + metrics map[string]string + metricsMutex sync.RWMutex +} + +// Creates new application-specific storage. +func newApplicationStorage() *applicationStorage { + as := &applicationStorage{} + as.initialize() + + return as +} + +// Initializes internal things. +func (as *applicationStorage) initialize() { + as.metrics = make(map[string]string) +} + +// Starts application-specific things, like goroutine for HTTP requests. +func (as *applicationStorage) start() {} diff --git a/internal/datastore/datastore.go b/internal/datastore/datastore.go new file mode 100644 index 0000000..c2bb65d --- /dev/null +++ b/internal/datastore/datastore.go @@ -0,0 +1,56 @@ +package datastore + +import ( + "context" + "log" + "sync" + + "go.dev.pztrn.name/metricator/internal/configuration" +) + +// DataStore is a data storage structure. It keeps all gathered metrics and gives +// them away on request. +type DataStore struct { + config *configuration.Config + ctx context.Context + doneChan chan struct{} + + datas map[string]*applicationStorage + datasMutex sync.RWMutex +} + +// NewDataStore creates new data storage. +func NewDataStore(ctx context.Context, cfg *configuration.Config) (*DataStore, chan struct{}) { + ds := &DataStore{ + config: cfg, + ctx: ctx, + doneChan: make(chan struct{}), + } + ds.initialize() + + return ds, ds.doneChan +} + +// Internal things initialization. +func (ds *DataStore) initialize() { + ds.datas = make(map[string]*applicationStorage) + + // Create applications defined in configuration. + + go func() { + <-ds.ctx.Done() + log.Println("Data storage stopped") + + ds.doneChan <- struct{}{} + }() +} + +// Start starts data storage asynchronous things. +func (ds *DataStore) Start() { + log.Println("Starting data storage...") + + ds.datasMutex.RLock() + for _, storage := range ds.datas { + storage.start() + } +} diff --git a/internal/httpserver/handler.go b/internal/httpserver/handler.go new file mode 100644 index 0000000..4501f49 --- /dev/null +++ b/internal/httpserver/handler.go @@ -0,0 +1,20 @@ +package httpserver + +import ( + "net/http" + + "go.dev.pztrn.name/metricator/internal/common" +) + +// HTTP requests handler. +type handler struct { + handler common.HTTPHandlerFunc +} + +// Registers request's handler. +func (h *handler) register(hndl common.HTTPHandlerFunc) { + h.handler = hndl +} + +// ServeHTTP handles every HTTP request. +func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {} diff --git a/internal/httpserver/httpserver.go b/internal/httpserver/httpserver.go new file mode 100644 index 0000000..a3ce0e8 --- /dev/null +++ b/internal/httpserver/httpserver.go @@ -0,0 +1,79 @@ +package httpserver + +import ( + "context" + "log" + "net" + "net/http" + "strings" + "time" + + "go.dev.pztrn.name/metricator/internal/configuration" +) + +// HTTPServer is a controlling structure for HTTP server. +type HTTPServer struct { + config *configuration.Config + ctx context.Context + doneChan chan struct{} + handler *handler + server *http.Server +} + +func NewHTTPServer(ctx context.Context, cfg *configuration.Config) (*HTTPServer, chan struct{}) { + h := &HTTPServer{ + config: cfg, + ctx: ctx, + doneChan: make(chan struct{}), + } + h.initialize() + + return h, h.doneChan +} + +// Returns request's context based on main context of application. +// Basically it returns main context and does nothing more. +func (h *HTTPServer) getRequestContext(_ net.Listener) context.Context { + return h.ctx +} + +// Initializes handler and HTTP server structure. +func (h *HTTPServer) initialize() { + h.handler = &handler{} + h.server = &http.Server{ + // ToDo: make it all configurable. + Addr: ":34421", + BaseContext: h.getRequestContext, + Handler: h.handler, + ReadTimeout: time.Second * 10, + WriteTimeout: time.Second * 10, + MaxHeaderBytes: 1 << 20, + } +} + +// Start starts HTTP server in another goroutine and one more goroutine which +// is listening to main context's Cancel() call to stop HTTP server. +func (h *HTTPServer) Start() { + go func() { + err := h.server.ListenAndServe() + if err != nil { + if !strings.Contains(err.Error(), "Server closed") { + log.Println("HTTP server failed to listen:", err.Error()) + } + } + }() + + go func() { + <-h.ctx.Done() + log.Println("Shutting down HTTP server") + + err := h.server.Shutdown(h.ctx) + if err != nil && !strings.Contains(err.Error(), "context canceled") { + log.Println("Failed to stop HTTP server:", err.Error()) + } + + log.Println("HTTP server stopped") + + h.doneChan <- struct{}{} + }() +} diff --git a/metricator.example.yaml b/metricator.example.yaml new file mode 100644 index 0000000..824c903 --- /dev/null +++ b/metricator.example.yaml @@ -0,0 +1,2 @@ +datastore: + valid_timeout: 30s diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..1978aef --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,17 @@ +# Scripts directory + +This directory contains various scripts that is used for whole application lifecycle - building, testing, deploying, etc. + +**WARNING**: these scripts assuming that they're launched from root directory of project! Do not launch them like: + +```shell +./some_stript.sh +``` + +Instead launch them from root directory like: + +```shell +./scripts/some_script.sh +``` + +Otherwise your computer might explode. You were warned. diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..24557cf --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# Metricator build script. + +source ./scripts/shell_helpers/get_git_data.sh diff --git a/scripts/shell_helpers/get_git_data.sh b/scripts/shell_helpers/get_git_data.sh new file mode 100644 index 0000000..54920aa --- /dev/null +++ b/scripts/shell_helpers/get_git_data.sh @@ -0,0 +1,7 @@ +# Gets git data. +# Should be sourced where neccessary. + +export BRANCHNAME=${BRANCHNAME:=$(git branch --no-color --show-current)} +export BUILDID=${BUILDID:=$(git rev-list HEAD --count)} +export COMMITHASH=${COMMITHASH:=$(git rev-parse --verify HEAD)} +export VERSION=${VERSION:="0.1.0-dev"} diff --git a/scripts/show_git_data.sh b/scripts/show_git_data.sh new file mode 100755 index 0000000..3b86e8d --- /dev/null +++ b/scripts/show_git_data.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +# Showing git data in console. +source ./scripts/shell_helpers/get_git_data.sh + +echo "* Branch: ${BRANCH}" +echo "* Build ID: ${BUILDID}" +echo "* Commit hash: ${COMMITHASH}" +echo "* Version: ${VERSION}"