Initial commit.

.gitignore

@@ -0,0 +1,2 @@
+._bin
+.vscode

Makefile

@@ -0,0 +1,26 @@
+# This is a Metricator Makefile.
+# It contains calls to scripts placed in scripts directory.
+
+help: Makefile
+	@echo -e "Metricator Makefile available subcommands:\n"
+	@cat $< | grep "## " | sort | sed -n 's/^## //p'
+	@echo ""
+	@make -f Makefile show-git-data
+
+.DEFAULT_GOAL := help
+
+check-build-dir:
+	@if [ ! -d "._bin" ]; then mkdir ._bin; fi
+
+## metricatord-build: builds metricator daemon and places into ${PWD}/._bin.
+metricatord-build: check-build-dir
+	rm ./._bin/metricatord || true
+	cd cmd/metricatord && go build -o ../../._bin/metricatord
+
+## metricatord-run: starts metricator daemon.
+metricatord-run: metricatord-build
+	./._bin/metricatord -config ./metricator.example.yaml
+
+show-git-data:
+	@echo "Parameters for current source code state:"
+	@scripts/show_git_data.sh

README.md

@@ -0,0 +1,26 @@
+# Metricator
+
+Simple proxy between prometheus-powered application and your NMS.
+
+## Why
+
+I'm using NetXMS to monitor all of my systems. Many things I use exports metrics in prometheus format which can be utilized by custom parsing script. But I've encounter a performance problem when I need to monitor 30 metrics - parsing script will make 30 requests to prometheus endpoint which might affect performance.
+
+Metricator will issue only one request and cache data in memory between them. Also it will expose HTTP API to get single metric which can be easily utilized with any NMS.
+
+## Caveats
+
+* No authorization. **DO NOT** expose Metricator to wild world!
+
+## Installation
+
+*TBW*
+
+## Configuration
+
+*TBW*
+
+
+## Documentation
+
+Check [docs directory](/docs/INDEX.md).

cmd/metricatord/main.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go.dev.pztrn.name/metricator/internal/common"
+	"go.dev.pztrn.name/metricator/internal/configuration"
+	"go.dev.pztrn.name/metricator/internal/datastore"
+	"go.dev.pztrn.name/metricator/internal/httpserver"
+)
+
+func main() {
+	log.Printf("Starting Metricator daemon, version %s from branch %s (build #%s, commit hash %s)\n",
+		common.Version,
+		common.Branch,
+		common.Build,
+		common.CommitHash,
+	)
+
+	mainCtx, cancelFunc := context.WithCancel(context.Background())
+	config := configuration.NewConfig()
+
+	httpSrv, httpStopped := httpserver.NewHTTPServer(mainCtx, config)
+	dataStore, dataStoreStopped := datastore.NewDataStore(mainCtx, config)
+
+	flag.Parse()
+	err := config.Parse()
+	if err != nil {
+		log.Fatalln("Failed to parse configuration:", err.Error())
+	}
+	log.Printf("Configuration parsed: %+v\n", config)
+
+	dataStore.Start()
+	httpSrv.Start()
+
+	log.Println("Metricator is started and ready to serve requests")
+
+	// CTRL+C handler.
+	signalHandler := make(chan os.Signal, 1)
+	shutdownDone := make(chan bool, 1)
+
+	signal.Notify(signalHandler, os.Interrupt, syscall.SIGTERM)
+
+	go func() {
+		<-signalHandler
+		cancelFunc()
+
+		<-dataStoreStopped
+		<-httpStopped
+
+		shutdownDone <- true
+	}()
+
+	<-shutdownDone
+	log.Println("Metricator stopped")
+	os.Exit(0)
+}

docs/INDEX.md

@@ -0,0 +1,13 @@
+# Metricator
+
+Welcome to Metricator docs.
+
+## What's the idea
+
+Main idea for Metricator is to provide high-speed proxy between classic NMSes (like NetXMS, Nagios and so on) and other software that exposes metrics in Prometheus format.
+
+Why proxy? Let's go by example.
+
+Imagine that software you wish to monitor with classic NMS exposes 250 metric items and you wrote a simple script which returns needed data. Classic NMSes is able to process only one value per checker (usually), which means that processing every metric will do 250 HTTP requests to monitored software which is obviosly not good.
+
+Metricator instead acts like Prometheus itself from monitored software PoV: it performs one HTTP request each timeout (configured in config), parses data and making it available to other requesters. As parsed metric data stored in memory it will be blazing fast and won't overload your system.

go.mod

@@ -0,0 +1,5 @@
+module go.dev.pztrn.name/metricator
+
+go 1.13
+
+require gopkg.in/yaml.v2 v2.3.0

go.sum

@@ -0,0 +1,3 @@
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

internal/common/const.go

@@ -0,0 +1,8 @@
+package common
+
+var (
+	Branch     string
+	Build      string
+	CommitHash string
+	Version    string
+)

internal/common/context_keys.go

@@ -0,0 +1,11 @@
+package common
+
+// ContextKey is a type of context.Context keys.
+type ContextKey string
+
+const (
+	// ContextKeyApplication specifies that returned value is a name of application.
+	ContextKeyApplication ContextKey = "applicationName"
+	// ContextKeyMetric specifies that returned value is a name of metric of application.
+	ContextKeyMetric ContextKey = "metricName"
+)

internal/common/http_handler.go

@@ -0,0 +1,6 @@
+package common
+
+import "context"
+
+// HTTPHandlerFunc describes signature of HTTP requests handling function.
+type HTTPHandlerFunc func(context.Context) string

internal/configuration/config.go

@@ -0,0 +1,93 @@
+package configuration
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	ErrConfigurationFileDoesNotExist  = errors.New("configuration file does not exist")
+	ErrConfigurationFilePathUndefined = errors.New("configuration file path wasn't provided")
+)
+
+// Config is an application's configuration.
+type Config struct {
+	configPath string
+	// Applications describes configuration for remote application's endpoints.
+	// Key is an application's name.
+	Applications map[string]struct {
+		// Endpoint is a remote application endpoint which should give us metrics
+		// in Prometheus format.
+		Endpoint string
+		// TimeBetweenRequests is a minimal amount of time which should pass
+		// between requests.
+		TimeBetweenRequests time.Duration
+	}
+	// Datastore describes data storage configuration.
+	Datastore struct {
+		// ValidTimeout is a timeout for which every data entry will be considered
+		// as valid. After that timeout if value wasn't updated it will be considered
+		// as invalid and purged from memory.
+		ValidTimeout time.Duration `yaml:"valid_timeout"`
+	} `yaml:"datastore"`
+}
+
+// NewConfig returns new configuration.
+func NewConfig() *Config {
+	c := &Config{}
+	c.initialize()
+
+	return c
+}
+
+// Initializes configuration.
+func (c *Config) initialize() {
+	flag.StringVar(&c.configPath, "config", "", "Configuration file path.")
+}
+
+// Parse parses configuration.
+func (c *Config) Parse() error {
+	if c.configPath == "" {
+		return ErrConfigurationFilePathUndefined
+	}
+
+	if strings.HasPrefix(c.configPath, "~") {
+		userDir, err := os.UserHomeDir()
+		if err != nil {
+			return fmt.Errorf("%s: %w", "file path normalization: getting user's home directory", err)
+		}
+
+		c.configPath = strings.Replace(c.configPath, "~", userDir, 1)
+	}
+
+	cfgPath, err := filepath.Abs(c.configPath)
+	if err != nil {
+		return fmt.Errorf("%s: %w", "file path normalization: getting absolute path", err)
+	}
+
+	c.configPath = cfgPath
+
+	if c.configPath == "" {
+		return fmt.Errorf("%s: %w", "file path normalization", ErrConfigurationFilePathUndefined)
+	}
+
+	fileData, err := ioutil.ReadFile(c.configPath)
+	if err != nil {
+		return fmt.Errorf("%s: %w", "configuration file read", err)
+	}
+
+	err = yaml.Unmarshal(fileData, c)
+	if err != nil {
+		return fmt.Errorf("%s: %w", "configuration file parsing", err)
+	}
+
+	return nil
+}

internal/datastore/application_store.go

@@ -0,0 +1,25 @@
+package datastore
+
+import "sync"
+
+// This is application-specific data storage.
+type applicationStorage struct {
+	metrics      map[string]string
+	metricsMutex sync.RWMutex
+}
+
+// Creates new application-specific storage.
+func newApplicationStorage() *applicationStorage {
+	as := &applicationStorage{}
+	as.initialize()
+
+	return as
+}
+
+// Initializes internal things.
+func (as *applicationStorage) initialize() {
+	as.metrics = make(map[string]string)
+}
+
+// Starts application-specific things, like goroutine for HTTP requests.
+func (as *applicationStorage) start() {}

internal/datastore/datastore.go

@@ -0,0 +1,56 @@
+package datastore
+
+import (
+	"context"
+	"log"
+	"sync"
+
+	"go.dev.pztrn.name/metricator/internal/configuration"
+)
+
+// DataStore is a data storage structure. It keeps all gathered metrics and gives
+//  them away on request.
+type DataStore struct {
+	config   *configuration.Config
+	ctx      context.Context
+	doneChan chan struct{}
+
+	datas      map[string]*applicationStorage
+	datasMutex sync.RWMutex
+}
+
+// NewDataStore creates new data storage.
+func NewDataStore(ctx context.Context, cfg *configuration.Config) (*DataStore, chan struct{}) {
+	ds := &DataStore{
+		config:   cfg,
+		ctx:      ctx,
+		doneChan: make(chan struct{}),
+	}
+	ds.initialize()
+
+	return ds, ds.doneChan
+}
+
+// Internal things initialization.
+func (ds *DataStore) initialize() {
+	ds.datas = make(map[string]*applicationStorage)
+
+	// Create applications defined in configuration.
+
+	go func() {
+		<-ds.ctx.Done()
+		log.Println("Data storage stopped")
+
+		ds.doneChan <- struct{}{}
+	}()
+}
+
+// Start starts data storage asynchronous things.
+func (ds *DataStore) Start() {
+	log.Println("Starting data storage...")
+
+	ds.datasMutex.RLock()
+	for _, storage := range ds.datas {
+		storage.start()
+	}
+}

internal/httpserver/handler.go

@@ -0,0 +1,20 @@
+package httpserver
+
+import (
+	"net/http"
+
+	"go.dev.pztrn.name/metricator/internal/common"
+)
+
+// HTTP requests handler.
+type handler struct {
+	handler common.HTTPHandlerFunc
+}
+
+// Registers request's handler.
+func (h *handler) register(hndl common.HTTPHandlerFunc) {
+	h.handler = hndl
+}
+
+// ServeHTTP handles every HTTP request.
+func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {}

internal/httpserver/httpserver.go

@@ -0,0 +1,79 @@
+package httpserver
+
+import (
+	"context"
+	"log"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	"go.dev.pztrn.name/metricator/internal/configuration"
+)
+
+// HTTPServer is a controlling structure for HTTP server.
+type HTTPServer struct {
+	config   *configuration.Config
+	ctx      context.Context
+	doneChan chan struct{}
+	handler  *handler
+	server   *http.Server
+}
+
+func NewHTTPServer(ctx context.Context, cfg *configuration.Config) (*HTTPServer, chan struct{}) {
+	h := &HTTPServer{
+		config:   cfg,
+		ctx:      ctx,
+		doneChan: make(chan struct{}),
+	}
+	h.initialize()
+
+	return h, h.doneChan
+}
+
+// Returns request's context based on main context of application.
+// Basically it returns main context and does nothing more.
+func (h *HTTPServer) getRequestContext(_ net.Listener) context.Context {
+	return h.ctx
+}
+
+// Initializes handler and HTTP server structure.
+func (h *HTTPServer) initialize() {
+	h.handler = &handler{}
+	h.server = &http.Server{
+		// ToDo: make it all configurable.
+		Addr:           ":34421",
+		BaseContext:    h.getRequestContext,
+		Handler:        h.handler,
+		ReadTimeout:    time.Second * 10,
+		WriteTimeout:   time.Second * 10,
+		MaxHeaderBytes: 1 << 20,
+	}
+}
+
+// Start starts HTTP server in another goroutine and one more goroutine which
+// is listening to main context's Cancel() call to stop HTTP server.
+func (h *HTTPServer) Start() {
+	go func() {
+		err := h.server.ListenAndServe()
+		if err != nil {
+			if !strings.Contains(err.Error(), "Server closed") {
+				log.Println("HTTP server failed to listen:", err.Error())
+			}
+		}
+	}()
+
+	go func() {
+		<-h.ctx.Done()
+		log.Println("Shutting down HTTP server")
+
+		err := h.server.Shutdown(h.ctx)
+		if err != nil && !strings.Contains(err.Error(), "context canceled") {
+			log.Println("Failed to stop HTTP server:", err.Error())
+		}
+
+		log.Println("HTTP server stopped")
+
+		h.doneChan <- struct{}{}
+	}()
+}

metricator.example.yaml

@@ -0,0 +1,2 @@
+datastore:
+  valid_timeout: 30s

scripts/README.md

@@ -0,0 +1,17 @@
+# Scripts directory
+
+This directory contains various scripts that is used for whole application lifecycle - building, testing, deploying, etc.
+
+**WARNING**: these scripts assuming that they're launched from root directory of project! Do not launch them like:
+
+```shell
+./some_stript.sh
+```
+
+Instead launch them from root directory like:
+
+```shell
+./scripts/some_script.sh
+```
+
+Otherwise your computer might explode. You were warned.

scripts/build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+# Metricator build script.
+
+source ./scripts/shell_helpers/get_git_data.sh

scripts/shell_helpers/get_git_data.sh

@@ -0,0 +1,7 @@
+# Gets git data.
+# Should be sourced where neccessary.
+
+export BRANCHNAME=${BRANCHNAME:=$(git branch --no-color --show-current)}
+export BUILDID=${BUILDID:=$(git rev-list HEAD --count)}
+export COMMITHASH=${COMMITHASH:=$(git rev-parse --verify HEAD)}
+export VERSION=${VERSION:="0.1.0-dev"}

scripts/show_git_data.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# Showing git data in console.
+source ./scripts/shell_helpers/get_git_data.sh
+
+echo "* Branch: ${BRANCH}"
+echo "* Build ID: ${BUILDID}"
+echo "* Commit hash: ${COMMITHASH}"
+echo "* Version: ${VERSION}"