12 Commits

Author SHA1 Message Date
e2a928fac4 Merge branch 'shanvl/bug_fix' into 'master'
Fix logger config and parser bugs; use scanner instead of reading request body at once

See merge request pztrn/metricator!1
2021-06-04 08:50:12 +00:00
Andrey Shcherbinin
a252681b26 Fix logger config and parser bugs; use scanner instead of reading request body at once 2021-06-04 08:50:12 +00:00
fb34d0d53e Version 0.2.0 and build fix. 2021-02-19 19:43:52 +05:00
abe6734a46 Preparing to 0.1.1 release. 2021-02-19 19:27:46 +05:00
c32d99ea65 Ability to overwrite go binary path when building.
Fixes #9.
2021-02-19 19:27:17 +05:00
edfbd5a90d Differentiate git/release data for binary data embedding.
Fixes #10.
2021-02-19 19:24:42 +05:00
10d761f07d Fix errors when building something with make.
Fixes #8.
2021-02-19 19:18:10 +05:00
ddf3ff9240 Documentation update and linting fix. 2020-12-24 23:33:24 +05:00
25d8b2776c Add metricator-client to docker image. 2020-12-24 23:17:38 +05:00
cdbb3b7089 Add possibility to format metricator client output.
Closes #5.
2020-12-24 23:15:56 +05:00
614526b16d The very basic metricator-client and package.
Package can be used in external things if needed.
2020-12-24 23:06:13 +05:00
f1418a7a31 Linting improvements. 2020-12-24 13:48:00 +05:00
24 changed files with 473 additions and 48 deletions

1
.gitignore vendored
View File

@@ -1,3 +1,4 @@
._bin
.vscode
metricator.yaml
.idea

View File

@@ -3,17 +3,10 @@ run:
linters:
enable-all: true
disable:
# Because globals might exist, but according to our codestyle they
# should be lowercased and considered as unexported.
- gochecknoglobals
# While it might be useful it'll create more problems that will solve.
- gocritic
# Complains about main() lengths, which isn't an issue.
- funlen
# Magic numbers might be everywhere. Disabled for now.
- gomnd
# ToDos everywhere
- godox
# Why? WHY? WHY _test???
- testpackage
linters-settings:
@@ -23,3 +16,13 @@ linters-settings:
min-complexity: 40
gocognit:
min-complexity: 40
funlen:
lines: 200
statements: 100
issues:
exclude-rules:
# There will be some ToDos.
- linters:
- godox
text: "TODO"

View File

@@ -11,11 +11,13 @@ ARG CI_COMMIT_TAG
ENV CGO_ENABLED=0
RUN apk add bash git make
RUN make metricatord-build
RUN make metricator-client-build
FROM registry.gitlab.pztrn.name/containers/mirror/golang:1.15.5-alpine
LABEL maintainer="Stanislav N. <pztrn@pztrn.name>"
COPY --from=build /go/src/gitlab.pztrn.name/pztrn/metricator/._bin/metricatord /usr/local/bin/metricatord
COPY --from=build /go/src/gitlab.pztrn.name/pztrn/metricator/._bin/metricator-client /usr/local/bin/metricator-client
RUN apk add tzdata

View File

@@ -14,11 +14,20 @@ help: Makefile
check-build-dir:
@if [ ! -d "._bin" ]; then mkdir ._bin; fi
## metricator-client-build: builds metricator client and places into ${PWD}/._bin.
metricator-client-build: check-build-dir
@if [ -f ./._bin/metricator-client ]; then rm ./._bin/metricator-client; fi
@scripts/build.sh metricator-client
## metricatord-build: builds metricator daemon and places into ${PWD}/._bin.
metricatord-build: check-build-dir
@rm ./._bin/metricatord || true
@if [ -f ./._bin/metricatord ]; then rm ./._bin/metricatord; fi
@scripts/build.sh metricatord
## metricator-client-run: starts metricator client. Use ARGS to supply args.
metricator-client-run: metricator-client-build
@./._bin/metricator-client -config ${CONFIG} $(ARGS)
## metricatord-run: starts metricator daemon.
metricatord-run: metricatord-build
./._bin/metricatord -config ${CONFIG}

1
VERSION Normal file
View File

@@ -0,0 +1 @@
0.2.0

View File

@@ -0,0 +1,156 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"log"
"os"
"go.dev.pztrn.name/metricator/internal/common"
"go.dev.pztrn.name/metricator/internal/configuration"
"go.dev.pztrn.name/metricator/internal/logger"
"go.dev.pztrn.name/metricator/pkg/client"
"go.dev.pztrn.name/metricator/pkg/schema"
)
// nolint:gochecknoglobals
var (
application = flag.String("application", "", "Application to query.")
appsList = flag.Bool("apps-list", false, "Show application's list registered at Metricator.")
metricatorHost = flag.String("metricator-host", "", "IP address or domain on which Metricator is available")
metricatorTimeout = flag.Int("metricator-timeout", 5, "Timeout for requests sent to Metricator.")
metricsList = flag.Bool("metrics-list", false, "Show metrics list. Requires 'application' parameter.")
metric = flag.String("metric", "", "Metric data to retrieve. Requires 'application' parameter.")
output = flag.String("output", "json", "Output format. Can be 'json' or 'plain-by-line'.")
)
func main() {
config := configuration.NewConfig()
// Parse configuration.
flag.Parse()
err := config.Parse()
if err != nil {
log.Fatalln("Failed to parse configuration:", err.Error())
}
logger := logger.NewLogger(config.Logger)
logger.Debugf("Starting Metricator client, version %s from branch %s (build #%s, commit hash %s)\n",
common.Version,
common.Branch,
common.Build,
common.CommitHash,
)
// Check configuration.
// We cannot work at all if host isn't defined.
if *metricatorHost == "" {
logger.Infoln("Host isn't defined.")
flag.PrintDefaults()
os.Exit(1)
}
// If nothing is requested - show error message.
if !*appsList && !*metricsList && *metric == "" {
logger.Infoln("No action specified.")
flag.PrintDefaults()
os.Exit(1)
}
// When asking to metrics list we need application to be defined.
if *metricsList && *application == "" {
logger.Infoln("Getting metrics list requires 'application' parameter to be filled.")
flag.PrintDefaults()
os.Exit(1)
}
// When asking for specific metric we need application to be defined.
if *metric != "" && *application == "" {
logger.Infoln("Getting metric data requires 'application' parameter to be filled.")
flag.PrintDefaults()
os.Exit(1)
}
clientConfig := &client.Config{
Host: *metricatorHost,
Timeout: *metricatorTimeout,
}
c := client.NewClient(clientConfig, logger)
var data interface{}
switch {
case *appsList:
data = c.GetAppsList()
case *metricsList:
data = c.GetMetricsList(*application)
case *metric != "":
data = c.GetMetric(*application, *metric)
}
switch *output {
case "json":
dataAsBytes, err := json.Marshal(data)
if err != nil {
logger.Infoln("Failed to marshal data from Metricator:", err.Error())
os.Exit(2)
}
fmt.Println(string(dataAsBytes))
case "plain-by-lines":
// For plain mode if we request metric - we should just print it and exit.
if *metric != "" {
fmt.Println(data)
os.Exit(0)
}
dataToPrint := []string{}
switch {
case *appsList:
appsListData, ok := data.(schema.AppsList)
if !ok {
logger.Infoln("Failed to cast parsed data into schema.AppsList!")
os.Exit(3)
}
for _, app := range appsListData {
dataToPrint = append(dataToPrint, app)
}
case *metric != "":
metricData, ok := data.(string)
if !ok {
logger.Infoln("Failed to cast parsed data into string!")
os.Exit(3)
}
dataToPrint = append(dataToPrint, metricData)
case *metricsList:
metricsData, ok := data.(schema.Metrics)
if !ok {
logger.Infoln("Failed to cast parsed data into schema.Metrics!")
os.Exit(3)
}
for _, metric := range metricsData {
dataToPrint = append(dataToPrint, metric.Name)
}
}
for _, line := range dataToPrint {
fmt.Println(line)
}
}
}

53
docs/CLIENT.md Normal file
View File

@@ -0,0 +1,53 @@
# Metricator Client
Metricator client was created to help with Metricator daemon communication. It is able to produce different output based on selected format.
## Parameters
| Parameter | Type | Description |
| --------- | ---- | ----------- |
| `-application` | string | Name of application to query. |
| `-apps-list` | bool | Request type: applications list. |
| `-config` | string | Path to configuration file. **MANDATORY** |
| `-metric` | string | Request type: single metric. Name of metric to request. |
| `-metricator-host` | URL | URL to Metricator daemon. (e.g. `http://127.0.0.1:34421`). **MANDATORY** |
| `-metricator-timeout` | integer | Timeout in seconds for Metricator Client's HTTP requests. By default - 5 seconds. |
| `-metrics-list` | bool | Request type: list of metrics. **Requires `-application` parameter to be filled.** |
| `-output` | string | Type of output to produce (see below). |
## Request types
One of following parameters should be defined:
* `-apps-list` to get listing of applications that is registered at Metricator daemon.
* `-metrics-list` with `-application` parameters to get list of metrics for application which Metricator know.
* `-metric` with name of metric and `-application` (with a name of application) parameters to get specific metric for application which Metricator know.
See Examples section below.
## Outputs
Currently Metricator client is able to produce JSON and "Plain By Line" outputs. Their meanings:
* When `-output=json` is specified (or `-output` wasn't specified at all) Metricator Client will just dump response from Metricator Daemon.
* When `-output=plain-by-line` is specified Metricator Client will transform received data into line-by-line output, e.g. every application name on separate line, every metric name on separate line, etc.
## Examples
* Get list of applications registered at Metricator daemon line-by-line (for later use with metrics autodiscovery helper):
```shell
metricator-client -config=./metricator.yaml -metricator-host http://127.0.0.1:34421 -output plain-by-line -apps-list
```
* Get list of metrics for application `test` in JSON format:
```shell
metricator-client -config ./metricator.yaml -metricator-host http://127.0.0.1:34421 -application test -metrics-list
```
* Get specific metric for application `test`:
```shell
metricator-client -config ./metricator.yaml -metricator-host http://127.0.0.1:34421 -application test -metric mymegametric
```

View File

@@ -31,3 +31,4 @@ dnsdist_frontend_responses/frontend:127.0.0.1:53/proto:UDP/thread:0
* [Installation](INSTALL.md)
* [Configuration](CONFIGURE.md)
* [API](API.md)
* [Client](CLIENT.md)

View File

@@ -1,7 +1,6 @@
package application
import (
"io/ioutil"
"net/http"
"time"
)
@@ -44,15 +43,13 @@ func (a *Application) fetch() {
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
data, err := a.parse(resp.Body)
if err != nil {
a.logger.Infoln("Failed to read response body for", a.name, "metrics:", err.Error())
a.logger.Infoln("Failed to parse response body for", a.name, "metrics:", err.Error())
return
}
data := a.parse(string(body))
a.storage.Put(data)
a.fetchIsRunningMutex.Lock()

View File

@@ -1,19 +1,27 @@
package application
import (
"bufio"
"fmt"
"io"
"strings"
"go.dev.pztrn.name/metricator/internal/models"
"go.dev.pztrn.name/metricator/pkg/schema"
)
// Parses passed body and returns a map suitable for pushing into storage.
func (a *Application) parse(body string) map[string]models.Metric {
data := make(map[string]models.Metric)
// Parses io.Reader passed and returns a map suitable for pushing into storage.
func (a *Application) parse(r io.Reader) (map[string]schema.Metric, error) {
data := make(map[string]schema.Metric)
// ToDo: switch to bytes buffer and maybe do not read body in caller?
splittedBody := strings.Split(body, "\n")
scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := scanner.Text()
// Skip empty lines.
if line == "" {
continue
}
for _, line := range splittedBody {
// Prometheus line contains metric name and metric parameters defined
// in "{}".
var (
@@ -21,11 +29,6 @@ func (a *Application) parse(body string) map[string]models.Metric {
params []string
)
// Skip empty lines.
if line == "" {
continue
}
a.logger.Debugln("Analyzing line:", line)
name = a.getMetricName(line)
@@ -35,7 +38,7 @@ func (a *Application) parse(body string) map[string]models.Metric {
if !found {
a.logger.Debugln("Metric wasn't yet created, creating new structure")
metric = models.NewMetric(name, "", "", nil)
metric = schema.NewMetric(name, "", "", nil)
}
a.logger.Debugf("Got metric to use: %+v\n", metric)
@@ -80,19 +83,22 @@ func (a *Application) parse(body string) map[string]models.Metric {
newMetric.Params = params
metric = newMetric
data[metric.Name] = metric
}
metric.Value = a.getMetricValue(line)
a.logger.Debugf("Got metric: %+v\n", metric)
data[name] = metric
data[metric.Name] = metric
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("wasn't able to scan input: %w", err)
}
a.logger.Debugf("Data parsed: %+v\n", data)
return data
return data, nil
}
// Gets metric description from passed line.

View File

@@ -1,5 +1,6 @@
package common
// nolint:gochecknoglobals
var (
Branch string
Build string

View File

@@ -13,6 +13,7 @@ import (
"go.dev.pztrn.name/metricator/internal/models"
)
// nolint:gochecknoglobals
var (
errInvalidAPIVersion = errors.New("invalid API version")
errInvalidApplication = errors.New("invalid application")

View File

@@ -9,6 +9,10 @@ type Logger struct {
// NewLogger creates new logging wrapper and returns it to caller.
func NewLogger(config *Config) *Logger {
if config == nil {
config = &Config{Debug: false}
}
l := &Logger{config: config}
return l

View File

@@ -1,19 +1,19 @@
package storage
import "go.dev.pztrn.name/metricator/internal/models"
import "go.dev.pztrn.name/metricator/pkg/schema"
// GenericStorage describes interface every other storage should embed
// and conform to as it contains essential things like context handling.
type GenericStorage interface {
// Get returns data from storage by key.
Get(string) (models.Metric, error)
Get(string) (schema.Metric, error)
// GetAsSlice returns all data from storage as slice.
GetAsSlice() []models.Metric
GetAsSlice() []schema.Metric
// GetDoneChan returns a channel which should be used to block execution
// until storage's routines are completed.
GetDoneChan() chan struct{}
// Put puts passed data into storage.
Put(map[string]models.Metric)
Put(map[string]schema.Metric)
// Start starts asynchronous things if needed.
Start()
}

View File

@@ -6,7 +6,7 @@ import (
"sync"
"go.dev.pztrn.name/metricator/internal/logger"
"go.dev.pztrn.name/metricator/internal/models"
"go.dev.pztrn.name/metricator/pkg/schema"
)
// ErrMetricNotFound appears if requested metric wasn't found in storage.
@@ -17,7 +17,7 @@ type Storage struct {
ctx context.Context
doneChan chan struct{}
logger *logger.Logger
data map[string]models.Metric
data map[string]schema.Metric
name string
dataMutex sync.RWMutex
}
@@ -36,7 +36,7 @@ func NewStorage(ctx context.Context, name string, logger *logger.Logger) (*Stora
}
// Get returns data from storage by key.
func (s *Storage) Get(key string) (models.Metric, error) {
func (s *Storage) Get(key string) (schema.Metric, error) {
s.logger.Debugln("Retrieving data for", key, "key from storage...")
s.dataMutex.RLock()
@@ -46,7 +46,7 @@ func (s *Storage) Get(key string) (models.Metric, error) {
if !found {
s.logger.Infoln("Key", key, "not found in storage!")
return models.NewMetric("", "", "", nil), ErrMetricNotFound
return schema.NewMetric("", "", "", nil), ErrMetricNotFound
}
s.logger.Debugf("Key %s found: %+v\n", key, data)
@@ -55,10 +55,10 @@ func (s *Storage) Get(key string) (models.Metric, error) {
}
// GetAsSlice returns all data from storage as slice.
func (s *Storage) GetAsSlice() []models.Metric {
func (s *Storage) GetAsSlice() []schema.Metric {
s.logger.Debugln("Returning all stored metrics as slice...")
metrics := make([]models.Metric, 0, len(s.data))
metrics := make([]schema.Metric, 0, len(s.data))
for _, metric := range s.data {
metrics = append(metrics, metric)
@@ -75,11 +75,11 @@ func (s *Storage) GetDoneChan() chan struct{} {
// Initializes internal things.
func (s *Storage) initialize() {
s.data = make(map[string]models.Metric)
s.data = make(map[string]schema.Metric)
}
// Put puts passed data into storage.
func (s *Storage) Put(data map[string]models.Metric) {
func (s *Storage) Put(data map[string]schema.Metric) {
s.dataMutex.Lock()
defer s.dataMutex.Unlock()

146
pkg/client/client.go Normal file
View File

@@ -0,0 +1,146 @@
package client
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"time"
"go.dev.pztrn.name/metricator/internal/logger"
"go.dev.pztrn.name/metricator/pkg/schema"
)
// Client is a Metricator client that is ready to be used in other applications
// or libraries.
type Client struct {
config *Config
logger *logger.Logger
httpClient *http.Client
}
// NewClient creates new Metricator client.
func NewClient(config *Config, logger *logger.Logger) *Client {
c := &Client{
config: config,
logger: logger,
}
c.initialize()
return c
}
// Executes request and parses it's contents.
func (c *Client) executeAndParse(req *http.Request, dest interface{}) error {
c.logger.Debugf("Executing HTTP request to %s%s", c.config.Host, req.URL.RequestURI())
ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*time.Duration(c.config.Timeout))
defer cancelFunc()
req = req.WithContext(ctx)
response, err := c.httpClient.Do(req)
if err != nil {
c.logger.Infoln("Failed to execute request to Metricator:", err.Error())
return fmt.Errorf("metricator client: %w", err)
}
defer response.Body.Close()
respData, err := ioutil.ReadAll(response.Body)
if err != nil {
c.logger.Infoln("Failed to read response body:", err.Error())
return fmt.Errorf("metricator client: %w", err)
}
err = json.Unmarshal(respData, dest)
if err != nil {
c.logger.Infoln("Failed to parse response:", err.Error())
return fmt.Errorf("metricator client: %w", err)
}
return nil
}
// GetAppsList returns a slice with applications that was registered at Metricator.
func (c *Client) GetAppsList() schema.AppsList {
address := fmt.Sprintf("%s/api/v1/apps_list", c.config.Host)
// Request's context sets in c.executeAndParse, so:
// nolint:noctx
req, err := http.NewRequest("GET", address, nil)
if err != nil {
c.logger.Infoln("Failed to create HTTP request:", err.Error())
return nil
}
appsList := make(schema.AppsList, 0)
err = c.executeAndParse(req, &appsList)
if err != nil {
return nil
}
return appsList
}
// GetMetric returns value for metric.
func (c *Client) GetMetric(appName, metricName string) interface{} {
address := fmt.Sprintf("%s/api/v1/metrics/%s/%s", c.config.Host, appName, metricName)
// Request's context sets in c.executeAndParse, so:
// nolint:noctx
req, err := http.NewRequest("GET", address, nil)
if err != nil {
c.logger.Infoln("Failed to create HTTP request:", err.Error())
return ""
}
var data interface{}
err = c.executeAndParse(req, &data)
if err != nil {
return ""
}
return data
}
// GetMetricsList returns a slice with metrics names for passed application.
func (c *Client) GetMetricsList(appName string) schema.Metrics {
address := fmt.Sprintf("%s/api/v1/metrics/%s", c.config.Host, appName)
// Request's context sets in c.executeAndParse, so:
// nolint:noctx
req, err := http.NewRequest("GET", address, nil)
if err != nil {
c.logger.Infoln("Failed to create HTTP request:", err.Error())
return nil
}
data := make(schema.Metrics, 0)
err = c.executeAndParse(req, &data)
if err != nil {
return nil
}
return data
}
// Initializes internal states and storages.
func (c *Client) initialize() {
// We do not need to set everything for client actually, so:
// nolint:exhaustivestruct
c.httpClient = &http.Client{
Timeout: time.Second * time.Duration(c.config.Timeout),
}
}

9
pkg/client/config.go Normal file
View File

@@ -0,0 +1,9 @@
package client
// Config is a Metricator client configuration.
type Config struct {
// Host is a host where Metricator is available for requests.
Host string
// Timeout specifies HTTP client timeout.
Timeout int
}

9
pkg/schema/apps_list.go Normal file
View File

@@ -0,0 +1,9 @@
package schema
// AppsList represents applications list structure from Metricator's API.
type AppsList []string
// IsEmpty returns true if returned applications list is empty.
func (a AppsList) IsEmpty() bool {
return len(a) == 0
}

View File

@@ -1,6 +1,6 @@
package models
package schema
// Metric is a generic metric structure.
// Metric is a generic metric structure. Used in HTTP responses and data storage.
type Metric struct {
// BaseName is a metric's base name, used for constructing name.
BaseName string

9
pkg/schema/metrics.go Normal file
View File

@@ -0,0 +1,9 @@
package schema
// Metrics is a metrics collection response.
type Metrics []*Metric
// IsEmpty returns true if returned applications list is empty.
func (m Metrics) IsEmpty() bool {
return len(m) == 0
}

View File

@@ -2,7 +2,13 @@
# Metricator build script.
GO=${GO:=$(which go)}
if [ -d .git ]; then
source ./scripts/shell_helpers/get_git_data.sh
else
source ./scripts/shell_helpers/get_release_data.sh
fi
WHATTOBUILD=$1
@@ -13,5 +19,7 @@ LINKERFLAGS="\
-X go.dev.pztrn.name/metricator/internal/common.Version=${VERSION}"
echo "Using $(go version) at ${GO}"
cd cmd/${WHATTOBUILD}
go build -tags netgo -ldflags "${LINKERFLAGS} -w -extldflags '-static'" -o ../../._bin/${WHATTOBUILD}
${GO} build -tags netgo -ldflags "${LINKERFLAGS} -w -extldflags '-static'" -o ../../._bin/${WHATTOBUILD}

View File

@@ -1,7 +1,6 @@
# Gets git data.
# Should be sourced where neccessary.
export BRANCHNAME=${BRANCHNAME:=$(git branch --no-color --show-current)}
export BUILDID=${BUILDID:=$(git rev-list HEAD --count)}
export COMMITHASH=${COMMITHASH:=$(git rev-parse --verify HEAD)}
export VERSION=${VERSION:="0.1.0-dev"}
export VERSION=${VERSION:=$(cat VERSION)}

View File

@@ -0,0 +1,6 @@
# Overrides some variables for release.
export VERSION=$(cat VERSION)
export BRANCHNAME="release/${VERSION}"
export COMMITHASH="none"
export BUILDID="0"

View File

@@ -1,7 +1,11 @@
#!/usr/bin/env bash
# Showing git data in console.
if [ -d .git ]; then
source ./scripts/shell_helpers/get_git_data.sh
else
source ./scripts/shell_helpers/get_release_data.sh
fi
echo "* Branch: ${BRANCHNAME}"
echo "* Build ID: ${BUILDID}"