Working with packages and allowed IPs.

giredorectl now able to interact with giredored about:

* Setting package data. There is no such thing as "create" or "update",
just set.

* Deleting package data.

* Setting allowed IP addresses. This is the only authorization method
ATM, more may come in future.

internal/configuration/config.go

@@ -1,22 +0,0 @@
-package configuration
-
-import (
-	// other
-	"github.com/vrischmann/envconfig"
-)
-
-type config struct {
-	HTTP struct {
-		Listen         string `envconfig:"default=127.0.0.1:62222"`
-		WaitForSeconds int    `envconfig:"default=10"`
-	}
-}
-
-// Initialize loads configuration into memory.
-func (cf *config) Initialize() {
-	log.Info().Msg("Loading configuration...")
-
-	_ = envconfig.Init(cf)
-
-	log.Info().Msgf("Configuration parsed: %+v", cf)
-}

internal/configuration/envconfig.go

@@ -0,0 +1,33 @@
+package configuration
+
+import (
+	// other
+	"github.com/vrischmann/envconfig"
+)
+
+// This structure represents configuration that will be parsed via
+// environment variables. This configuration has higher priority
+// than configuration loaded from file.
+type envConfig struct {
+	// DataDir is a directory where giredore will store it's data
+	// like dynamic configuration file.
+	DataDir string `envconfig:"default=/var/lib/giredore"`
+	// HTTP describes HTTP server configuration.
+	HTTP struct {
+		// Listen is an address on which HTTP server will listen.
+		Listen string `envconfig:"default=127.0.0.1:62222"`
+		// WaitForSeconds is a timeout during which we will wait for
+		// HTTP server be up. If timeout will pass and HTTP server won't
+		// start processing requests - giredore will exit.
+		WaitForSeconds int `envconfig:"default=10"`
+	}
+}
+
+// Initialize parses environment variables into structure.
+func (cf *envConfig) Initialize() {
+	log.Info().Msg("Loading configuration...")
+
+	_ = envconfig.Init(cf)
+
+	log.Info().Msgf("Environment parsed: %+v", cf)
+}

internal/configuration/exported.go

@@ -12,7 +12,8 @@ var (
 	log               zerolog.Logger
 	loggerInitialized bool
 
-	Cfg *config
+	envCfg *envConfig
+	Cfg    *fileConfig
 )
 
 func Initialize() {
@@ -20,6 +21,16 @@ func Initialize() {
 	loggerInitialized = true
 	log.Info().Msg("Initializing...")
 
-	Cfg = &config{}
+	envCfg = &envConfig{}
+	envCfg.Initialize()
+
+	Cfg = &fileConfig{}
 	Cfg.Initialize()
+
+	Cfg.HTTP.Listen = envCfg.HTTP.Listen
+	Cfg.HTTP.WaitForSeconds = envCfg.HTTP.WaitForSeconds
+}
+
+func Shutdown() {
+	Cfg.Save()
 }

internal/configuration/fileconfig.go

@@ -0,0 +1,204 @@
+package configuration
+
+import (
+	// stdlib
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	// local
+	"sources.dev.pztrn.name/pztrn/giredore/internal/structs"
+)
+
+// This structure represents configuration that will be parsed via file.
+// Despite on exporting fields there are setters and getters defined because
+// data from configuration file will be parsed in exported fields and they
+// may be accesses concurrently. In other words DO NOT USE EXPORTED FIELDS
+// DIRECTLY!
+type fileConfig struct {
+	// HTTP describes HTTP server configuration.
+	HTTP struct {
+		// AllowedIPs is a list of IPs that allowed to access API.
+		// There might be other authentication implemented in future.
+		AllowedIPs      []string
+		allowedipsmutex sync.RWMutex
+		// Listen is an address on which HTTP server will listen.
+		Listen string `envconfig:"default=127.0.0.1:62222"`
+		// WaitForSeconds is a timeout during which we will wait for
+		// HTTP server be up. If timeout will pass and HTTP server won't
+		// start processing requests - giredore will exit.
+		WaitForSeconds int `envconfig:"default=10"`
+	}
+	// Packages describes packages mapping.
+	Packages      map[string]*structs.Package
+	packagesMutex sync.RWMutex
+}
+
+func (fc *fileConfig) AddOrUpdatePackage(pkg *structs.Package) {
+	fc.packagesMutex.Lock()
+	fc.Packages[pkg.OriginalPath] = pkg
+	fc.packagesMutex.Unlock()
+}
+
+func (fc *fileConfig) DeletePackage(req *structs.PackageDeleteRequest) []structs.Error {
+	var errors []structs.Error
+
+	fc.packagesMutex.Lock()
+	defer fc.packagesMutex.Unlock()
+	_, found := fc.Packages[req.OriginalPath]
+
+	if !found {
+		errors = append(errors, structs.ErrPackageWasntDefined)
+		return errors
+	}
+
+	delete(fc.Packages, req.OriginalPath)
+
+	return errors
+}
+
+func (fc *fileConfig) GetAllowedIPs() []string {
+	var allowedIPs []string
+	fc.HTTP.allowedipsmutex.RLock()
+	allowedIPs = append(allowedIPs, fc.HTTP.AllowedIPs...)
+	fc.HTTP.allowedipsmutex.RUnlock()
+
+	return allowedIPs
+}
+
+func (fc *fileConfig) GetAllPackagesInfo() map[string]*structs.Package {
+	pkgs := make(map[string]*structs.Package)
+
+	fc.packagesMutex.Lock()
+	for name, pkg := range fc.Packages {
+		pkgs[name] = pkg
+	}
+	fc.packagesMutex.Unlock()
+
+	return pkgs
+}
+
+func (fc *fileConfig) GetPackagesInfo(packages []string) (map[string]*structs.Package, []structs.Error) {
+	pkgs := make(map[string]*structs.Package)
+	var errors []structs.Error
+
+	fc.packagesMutex.Lock()
+	for _, neededPkg := range packages {
+		pkgData, found := fc.Packages[neededPkg]
+		if !found {
+			errors = append(errors, structs.ErrPackageWasntDefined+structs.Error(" Package was: "+neededPkg))
+		} else {
+			pkgs[neededPkg] = pkgData
+		}
+	}
+	fc.packagesMutex.Unlock()
+
+	return pkgs, errors
+}
+
+// Initialize parses file contents into structure.
+func (fc *fileConfig) Initialize() {
+	configPath := filepath.Join(envCfg.DataDir, "config.json")
+	cfgLoadLog := log.With().Str("configuration path", configPath).Logger()
+	cfgLoadLog.Info().Msg("Loading configuration file...")
+
+	configPath, err := fc.normalizePath(configPath)
+	if err != nil {
+		cfgLoadLog.Fatal().Err(err).Msg("Failed to normalize configuration file path.")
+	}
+
+	// Check if file "config.json" specified in envConfig.DataDir field
+	// exists.
+	if _, err2 := os.Stat(configPath); os.IsNotExist(err2) {
+		cfgLoadLog.Error().Msg("Unable to load configuration from filesystem.")
+		return
+	}
+
+	// Try to load file into memory.
+	fileData, err3 := ioutil.ReadFile(configPath)
+	if err3 != nil {
+		cfgLoadLog.Fatal().Err(err3).Msg("Failed to read configuration file data into memory.")
+	}
+
+	// ...and parse it.
+	err4 := json.Unmarshal(fileData, fc)
+	if err4 != nil {
+		cfgLoadLog.Fatal().Err(err4).Msg("Failed to parse configuration file.")
+	}
+
+	if fc.Packages == nil {
+		fc.Packages = make(map[string]*structs.Package)
+	}
+
+	// Ensure that localhost (127.0.0.1) are defined in AllowedIPs.
+	var localhostIsAllowed bool
+	for _, ip := range fc.HTTP.AllowedIPs {
+		if strings.Contains(ip, "127.0.0.1") {
+			localhostIsAllowed = true
+			break
+		}
+	}
+
+	if !localhostIsAllowed {
+		cfgLoadLog.Warn().Msg("Localhost (127.0.0.1) wasn't allowed to access configuration API, adding it to list of allowed IP addresses")
+		fc.HTTP.AllowedIPs = append(fc.HTTP.AllowedIPs, "127.0.0.1")
+	} else {
+		cfgLoadLog.Debug().Msg("Localhost (127.0.0.1) is allowed to access configuration API")
+	}
+
+	cfgLoadLog.Debug().Msgf("Configuration parsed: %+v", fc)
+	cfgLoadLog.Info().Int("packages count", len(fc.Packages)).Msg("Packages list loaded")
+}
+
+// Normalizes passed configuration file path.
+func (fc *fileConfig) normalizePath(configPath string) (string, error) {
+	// Normalize configuration file path.
+	if strings.Contains(configPath, "~") {
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			return "", err
+		}
+		configPath = strings.Replace(configPath, "~", homeDir, 1)
+	}
+
+	absPath, err1 := filepath.Abs(configPath)
+	if err1 != nil {
+		return "", err1
+	}
+
+	return absPath, nil
+}
+
+// Save saves configuration into file.
+func (fc *fileConfig) Save() {
+	configPath := filepath.Join(envCfg.DataDir, "config.json")
+	cfgSaveLog := log.With().Str("configuration path", configPath).Logger()
+	cfgSaveLog.Info().Msg("Saving configuration file...")
+
+	data, err := json.Marshal(fc)
+	if err != nil {
+		cfgSaveLog.Fatal().Err(err).Msg("Failed to encode data into JSON. Configuration file won't be saved!")
+		return
+	}
+
+	configPath, err1 := fc.normalizePath(configPath)
+	if err1 != nil {
+		cfgSaveLog.Fatal().Err(err1).Msg("Failed to normalize configuration file path.")
+	}
+
+	err2 := ioutil.WriteFile(configPath, data, os.ModePerm)
+	if err2 != nil {
+		cfgSaveLog.Fatal().Err(err2).Msg("Failed to write configuration file data to file.")
+	}
+
+	cfgSaveLog.Info().Msg("Configuration file saved.")
+}
+
+func (fc *fileConfig) SetAllowedIPs(allowedIPs []string) {
+	fc.HTTP.allowedipsmutex.Lock()
+	fc.HTTP.AllowedIPs = allowedIPs
+	fc.HTTP.allowedipsmutex.Unlock()
+}

internal/httpserver/checkallowedips.go

@@ -0,0 +1,65 @@
+package httpserver
+
+import (
+	// stdlib
+	"net"
+	"net/http"
+	"strings"
+
+	// local
+	"sources.dev.pztrn.name/pztrn/giredore/internal/configuration"
+	"sources.dev.pztrn.name/pztrn/giredore/internal/structs"
+
+	// other
+	"github.com/labstack/echo"
+)
+
+func checkAllowedIPs() echo.MiddlewareFunc {
+	return func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(ec echo.Context) error {
+			// Do nothing if request came not in "/_api" namespace.
+			if !strings.HasPrefix(ec.Request().RequestURI, "/_api") {
+				_ = next(ec)
+				return nil
+			}
+
+			// Get IPs and subnets from configuration and parse them
+			// into comparable things.
+			// If IP address was specified without network mask - assume /32.
+			var subnets []*net.IPNet
+			allowedIPs := configuration.Cfg.GetAllowedIPs()
+			for _, ip := range allowedIPs {
+				ipToParse := ip
+				if !strings.Contains(ip, "/") {
+					ipToParse = ip + "/32"
+				}
+
+				_, net, err := net.ParseCIDR(ipToParse)
+				if err != nil {
+					log.Error().Err(err).Str("subnet", ipToParse).Msg("Failed to parse CIDR. /_api/ endpoint won't be accessible, this should be fixed manually in configuration file!")
+					return ec.JSON(http.StatusInternalServerError, &structs.Reply{Status: structs.StatusFailure, Errors: []structs.Error{structs.ErrInvalidAllowedIPDefined}})
+				}
+
+				subnets = append(subnets, net)
+			}
+
+			// Check if requester's IP address are within allowed IP
+			// subnets.
+			ipToCheck := net.ParseIP(ec.RealIP())
+			var allowed bool
+			for _, subnet := range subnets {
+				if subnet.Contains(ipToCheck) {
+					allowed = true
+					break
+				}
+			}
+
+			if allowed {
+				_ = next(ec)
+				return nil
+			}
+
+			return ec.JSON(http.StatusBadRequest, &structs.Reply{Status: structs.StatusFailure, Errors: []structs.Error{structs.ErrIPAddressNotAllowed}})
+		}
+	}
+}

internal/httpserver/exported.go

@@ -32,9 +32,11 @@ func Initialize() {
 	Srv = echo.New()
 	Srv.Use(middleware.Recover())
 	Srv.Use(requestLogger())
+	Srv.Use(checkAllowedIPs())
 	Srv.DisableHTTP2 = true
 	Srv.HideBanner = true
 	Srv.HidePort = true
+	Srv.Binder = echo.Binder(&StrictJSONBinder{})
 
 	Srv.GET("/_internal/waitForOnline", waitForHTTPServerToBeUpHandler)
 }

internal/httpserver/strictjson.go

@@ -0,0 +1,38 @@
+package httpserver
+
+import (
+	// stdlib
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	// other
+	"github.com/labstack/echo"
+)
+
+// StrictJSONBinder implements Binder interface for Echo. It will parse
+// JSON in strict mode throwing errors on schema mismatches.
+type StrictJSONBinder struct{}
+
+// Bind parses JSON input.
+func (sjb *StrictJSONBinder) Bind(i interface{}, c echo.Context) error {
+	req := c.Request()
+	if req.ContentLength == 0 {
+		return echo.NewHTTPError(http.StatusBadRequest, "Request body can't be empty")
+	}
+
+	// Decode it.
+	decoder := json.NewDecoder(req.Body)
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(i); err != nil {
+		if ute, ok := err.(*json.UnmarshalTypeError); ok {
+			return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unmarshal type error: expected=%v, got=%v, field=%v, offset=%v", ute.Type, ute.Value, ute.Field, ute.Offset))
+		} else if se, ok := err.(*json.SyntaxError); ok {
+			return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Syntax error: offset=%v, error=%v", se.Offset, se.Error()))
+		} else {
+			return echo.NewHTTPError(http.StatusBadRequest, err.Error())
+		}
+	}
+
+	return nil
+}

internal/requester/exported.go

@@ -2,6 +2,8 @@ package requester
 
 import (
 	// stdlib
+	"bytes"
+	"encoding/json"
 	"io/ioutil"
 	"net/http"
 
@@ -21,14 +23,22 @@ func Initialize() {
 	log.Info().Msg("Initializing...")
 }
 
-func execRequest(method string, url string, data map[string]string) ([]byte, error) {
+func Delete(url string, data interface{}) ([]byte, error) {
+	return execRequest("DELETE", url, data)
+}
+
+func execRequest(method string, url string, data interface{}) ([]byte, error) {
 	log.Debug().Str("method", method).Str("URL", url).Msg("Trying to execute HTTP request...")
 
 	httpClient := getHTTPClient()
 
+	var dataToSend []byte
+	if data != nil {
+		dataToSend, _ = json.Marshal(data)
+	}
+
 	// Compose HTTP request.
-	// ToDo: POST/PUT/other methods that require body.
-	httpReq, err := http.NewRequest(method, url, nil)
+	httpReq, err := http.NewRequest(method, url, bytes.NewReader(dataToSend))
 	if err != nil {
 		return nil, err
 	}
@@ -52,3 +62,11 @@ func execRequest(method string, url string, data map[string]string) ([]byte, err
 func Get(url string) ([]byte, error) {
 	return execRequest("GET", url, nil)
 }
+
+func Post(url string, data interface{}) ([]byte, error) {
+	return execRequest("POST", url, data)
+}
+
+func Put(url string, data interface{}) ([]byte, error) {
+	return execRequest("PUT", url, data)
+}

internal/structs/allowedips.go

@@ -0,0 +1,5 @@
+package structs
+
+type AllowedIPsSetRequest struct {
+	AllowedIPs []string
+}

internal/structs/error.go

@@ -0,0 +1,12 @@
+package structs
+
+const (
+	ErrInvalidAllowedIPDefined     Error = "Invalid allowed IP address defined."
+	ErrIPAddressNotAllowed         Error = "IP address not allowed to access configuration API."
+	ErrPackageWasntDefined         Error = "Passed package wasn't defined."
+	ErrParsingAllowedIPsSetRequest Error = "Error parsing allowed IPs request."
+	ErrParsingDeleteRequest        Error = "Delete request parsing failed"
+	ErrParsingPackagesGetRequest   Error = "Error parsing package(s) info get request"
+)
+
+type Error string

internal/structs/package.go

@@ -0,0 +1,37 @@
+package structs
+
+// Package defines structure for 'pkg set' request and for storing it's
+// data in configuration.
+type Package struct {
+	// Description is an additional and optional description that
+	// can be show on package's page.
+	Description string
+	// OriginalPath is a package original path without domain part.
+	// E.g. for package "go.example.tld/group/pkgname" you should
+	// put here "/group/pkgname".
+	OriginalPath string
+	// RealPath is a path where package will be found. It should
+	// contain VCS path, e.g. "https://github.com/user/project.git".
+	RealPath string
+	// VCS is a versioning control system used for package. Everything
+	// that is supported by "go get" is applicable.
+	VCS string
+}
+
+// PackageDeleteRequest defines structure for package deleting request.
+type PackageDeleteRequest struct {
+	// OriginalPath is a package original path without domain part.
+	// E.g. for package "go.example.tld/group/pkgname" you should
+	// put here "/group/pkgname".
+	OriginalPath string
+}
+
+// PackageGetRequest defined structure for package information getting
+// request.
+type PackageGetRequest struct {
+	// Should all packages be returned?
+	All bool
+	// If All = false, then what package name (or names) to return?
+	// They should be delimited with comma in CLI.
+	PackageNames []string
+}

internal/structs/reply.go

@@ -0,0 +1,9 @@
+package structs
+
+// Reply defined reply data structure that giredored and giredorectl
+// will use.
+type Reply struct {
+	Status Status
+	Errors []Error
+	Data   interface{}
+}

internal/structs/status.go

@@ -0,0 +1,8 @@
+package structs
+
+const (
+	StatusFailure Status = "failure"
+	StatusSuccess Status = "success"
+)
+
+type Status string