Initial commit.

parsers/exported.go

@@ -0,0 +1,66 @@
+package parsers
+
+import (
+	// stdlib
+	"errors"
+	"log"
+	"sync"
+
+	// local
+	"go.dev.pztrn.name/glp/configuration"
+	"go.dev.pztrn.name/glp/parsers/golang"
+	"go.dev.pztrn.name/glp/parsers/parserinterface"
+	"go.dev.pztrn.name/glp/structs"
+)
+
+var (
+	parsers      map[string]parserinterface.Interface
+	parsersMutex sync.RWMutex
+)
+
+// Initialize initializes package.
+func Initialize() {
+	log.Println("Initializing parsers...")
+
+	parsers = make(map[string]parserinterface.Interface)
+
+	// Initialize parsers.
+	golangIface, golangName := golang.Initialize()
+	parsers[golangName] = golangIface
+}
+
+// Detect tries to launch parsers for project detection. It returns
+// parser name that should be used and optional flavor (e.g. dependencies
+// manager name) that might be returned by parser's Detect() function.
+func Detect(pkgPath string) (string, string) {
+	parsersMutex.RLock()
+	defer parsersMutex.RUnlock()
+
+	for parserName, parserIface := range parsers {
+		if configuration.Cfg.Log.Debug {
+			log.Println("Checking if parser '" + parserName + "' can parse project '" + pkgPath + "'...")
+		}
+
+		useThisParser, flavor := parserIface.Detect(pkgPath)
+		if useThisParser {
+			return parserName, flavor
+		}
+	}
+
+	return "unknown", ""
+}
+
+// GetDependencies asks parser to extract dependencies from project.
+func GetDependencies(parserName string, flavor string, pkgPath string) ([]*structs.Dependency, error) {
+	parsersMutex.RLock()
+	defer parsersMutex.RUnlock()
+	parser, found := parsers[parserName]
+
+	if !found {
+		return nil, errors.New("parser with such name isn't registered")
+	}
+
+	deps := parser.GetDependencies(flavor, pkgPath)
+
+	return deps, nil
+}

parsers/golang/dep.go

@@ -0,0 +1,109 @@
+package golang
+
+import (
+	// stdlib
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	// local
+	"go.dev.pztrn.name/glp/configuration"
+	"go.dev.pztrn.name/glp/structs"
+
+	// other
+	"github.com/BurntSushi/toml"
+)
+
+var goDepFilesToCheck = []string{"Gopkg.toml", "Gopkg.lock"}
+
+type depLockConfig struct {
+	Projects []struct {
+		Branch    string
+		Digest    string
+		Name      string
+		Packages  []string
+		PruneOpts string
+		Revision  string
+		Version   string
+	}
+	SolveMeta struct {
+		AnalyzerName    string   `toml:"analyzer-name"`
+		AnalyzerVersion int      `toml:"analyzer-version"`
+		InputImports    []string `toml:"input-imports"`
+		SolverName      string   `toml:"solver-name"`
+		SolverVersion   int      `toml:"solver-version"`
+	} `toml:"solve-meta"`
+}
+
+// Detects if project is using dep for dependencies management.
+func (gp *golangParser) detectDepUsage(pkgPath string) bool {
+	var goDepFilesFound bool
+	for _, fileName := range goDepFilesToCheck {
+		pathToCheck := filepath.Join(pkgPath, fileName)
+		if _, err := os.Stat(pathToCheck); err == nil {
+			goDepFilesFound = true
+		}
+	}
+
+	if goDepFilesFound {
+		log.Println("Project '" + pkgPath + "' is using dep for dependencies management")
+	}
+
+	return goDepFilesFound
+}
+
+// Gets dependencies data from dep-enabled projects.
+func (gp *golangParser) getDependenciesFromDep(pkgPath string) []*structs.Dependency {
+	deps := make([]*structs.Dependency, 0)
+
+	// Try to figure out parent package name for all dependencies.
+	parent := gp.getParentForDep(pkgPath)
+
+	// All dependencies for project will be taken from Gopkg.lock file.
+	lockFile := &depLockConfig{}
+	_, err := toml.DecodeFile(filepath.Join(pkgPath, "Gopkg.lock"), lockFile)
+	if err != nil {
+		log.Fatalln("Failed to parse dep lock file:", err.Error())
+	}
+
+	if configuration.Cfg.Log.Debug {
+		log.Printf("dep lock file parsed: %+v\n", lockFile)
+	}
+
+	// Parse dependencies.
+	for _, dep := range lockFile.Projects {
+		dependency := &structs.Dependency{
+			Name:   dep.Name,
+			Parent: parent,
+			VCS: structs.VCSData{
+				Branch:   dep.Branch,
+				Revision: dep.Revision,
+			},
+			Version: dep.Version,
+		}
+
+		// If branch is empty - assume master.
+		if dependency.VCS.Branch == "" {
+			dependency.VCS.Branch = "master"
+		}
+
+		deps = append(deps, dependency)
+
+		if configuration.Cfg.Log.Debug {
+			log.Printf("Initial dependency structure formed: %+v\n", dependency)
+		}
+	}
+
+	return deps
+}
+
+// Tries to get package name for passed package path.
+func (gp *golangParser) getParentForDep(pkgPath string) string {
+	// Dep-managed projects are in 99% of cases are placed in GOPATH.
+	if strings.Contains(pkgPath, "src") {
+		return strings.Split(pkgPath, "src/")[1]
+	}
+
+	return ""
+}

parsers/golang/exported.go

@@ -0,0 +1,16 @@
+package golang
+
+import (
+	// stdlib
+	"log"
+
+	// local
+	"go.dev.pztrn.name/glp/parsers/parserinterface"
+)
+
+func Initialize() (parserinterface.Interface, string) {
+	log.Println("Initializing Golang projects parser")
+
+	p := &golangParser{}
+	return parserinterface.Interface(p), "golang"
+}

parsers/golang/godata.go

@@ -0,0 +1,132 @@
+package golang
+
+import (
+	// stdlib
+	"bytes"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+
+	// local
+	"go.dev.pztrn.name/glp/configuration"
+	"go.dev.pztrn.name/glp/httpclient"
+	"go.dev.pztrn.name/glp/structs"
+)
+
+// Gets go-import and go-source data and fill it in dependency.
+func getGoData(dependency *structs.Dependency) {
+	// Dependencies are imported using URL which can be called with
+	// "?go-get=1" parameter to obtain required VCS data.
+	req, _ := http.NewRequest("GET", "http://"+dependency.Name, nil)
+
+	q := req.URL.Query()
+	q.Add("go-get", "1")
+
+	req.URL.RawQuery = q.Encode()
+
+	respBody := httpclient.GET(req)
+	if respBody == nil {
+		return
+	}
+
+	// HTML is hard to parse properly statically, so we will go
+	// line-by-line for <head> parsing.
+	resp := bytes.NewBuffer(respBody)
+
+	var (
+		// This flag shows that we're currently parsing <head> from HTML.
+		headCurrentlyParsing bool
+	)
+
+	for {
+		line, err := resp.ReadString('\n')
+		if err != nil && err != io.EOF {
+			log.Fatalln("Failed to read HTML response line-by-line:", err.Error())
+		} else if err != nil && err == io.EOF {
+			break
+		}
+
+		if headCurrentlyParsing {
+			// Check for go-import data.
+			if strings.Contains(line, `<meta name="go-import"`) {
+				// Get content.
+				// Import things are in element #4.
+				lineSplitted := strings.Split(line, `"`)
+
+				// Check line length. This is not so good approach, but
+				// should work for 99% of dependencies.
+				if len(lineSplitted) < 5 {
+					log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - tag itself appears to be incomplete. Skipping")
+					continue
+				}
+
+				if len(lineSplitted) > 5 {
+					log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - line where meta tag is located appears to be too long. Skipping")
+					continue
+				}
+
+				// Import line contains data like VCS name and VCS URL.
+				// They're delimited with whitespace.
+				importDataSplitted := strings.Split(lineSplitted[3], " ")
+
+				// Import line should contain at least 3 elements.
+				if len(importDataSplitted) < 3 {
+					log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - import data is too small. Skipping")
+					continue
+				}
+
+				// Fill dependency data with this data.
+				// First element is a module name and we do not actually
+				// need it, because it is already filled previously.
+				dependency.VCS.VCS = importDataSplitted[1]
+				dependency.VCS.VCSPath = importDataSplitted[2]
+			}
+
+			// Check for go-source data.
+			if strings.Contains(line, `<meta name="go-source"`) {
+				// Get content.
+				// Import things are in element #4.
+				lineSplitted := strings.Split(line, `"`)
+
+				// Check line length. This is not so good approach, but
+				// should work for 99% of dependencies.
+				if len(lineSplitted) < 5 {
+					log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - tag itself appears to be incomplete. Skipping")
+					continue
+				}
+
+				if len(lineSplitted) > 5 {
+					log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - line where meta tag is located appears to be too long. Skipping")
+					continue
+				}
+
+				// Source line contains data like VCS paths templates.
+				// They're delimited with whitespace.
+				sourceDataSplitted := strings.Split(lineSplitted[3], " ")
+
+				// Source data line should contain at least 3 elements.
+				if len(sourceDataSplitted) < 4 {
+					log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - source data is too small. Skipping")
+					continue
+				}
+
+				// Fill dependency data.
+				dependency.VCS.SourceURLDirTemplate = sourceDataSplitted[2]
+				dependency.VCS.SourceURLFileTemplate = sourceDataSplitted[3]
+			}
+		}
+
+		if strings.Contains(strings.ToLower(line), "<head>") {
+			headCurrentlyParsing = true
+		}
+
+		if strings.Contains(strings.ToLower(line), "</head>") {
+			headCurrentlyParsing = false
+		}
+	}
+
+	if configuration.Cfg.Log.Debug {
+		log.Printf("go-import and go-source data parsed: %+v\n", dependency.VCS)
+	}
+}

parsers/golang/modules.go

@@ -0,0 +1,27 @@
+package golang
+
+import (
+	// stdlib
+	"log"
+	"os"
+	"path/filepath"
+)
+
+var goModulesFilesToCheck = []string{"go.mod", "go.sum"}
+
+// Detects if project is using go modules for dependencies management.
+func (gp *golangParser) detectModulesUsage(pkgPath string) bool {
+	var goModulesFileFound bool
+	for _, fileName := range goModulesFilesToCheck {
+		pathToCheck := filepath.Join(pkgPath, fileName)
+		if _, err := os.Stat(pathToCheck); err == nil {
+			goModulesFileFound = true
+		}
+	}
+
+	if goModulesFileFound {
+		log.Println("Project '" + pkgPath + "' is using Go modules for dependencies management")
+	}
+
+	return goModulesFileFound
+}

parsers/golang/parser.go

@@ -0,0 +1,67 @@
+package golang
+
+import (
+	// stdlib
+	"sync"
+
+	// local
+	"go.dev.pztrn.name/glp/structs"
+)
+
+const (
+	// Package managers names. Used in Detect() for flavor returning.
+	packageManagerGoMod = "go mod"
+	packageManagerDep   = "dep"
+)
+
+// This structure responsible for parsing projects that written in Go.
+type golangParser struct{}
+
+// Detect detects if passed project path can be parsed with this parser
+// and additionally detect package manager used.
+func (gp *golangParser) Detect(pkgPath string) (bool, string) {
+	// Go projects usually using go modules or dep for dependencies
+	// management.
+	isModules := gp.detectModulesUsage(pkgPath)
+	if isModules {
+		return true, packageManagerGoMod
+	}
+
+	isDep := gp.detectDepUsage(pkgPath)
+	if isDep {
+		return true, packageManagerDep
+	}
+
+	return false, ""
+}
+
+// GetDependencies extracts dependencies from project.
+func (gp *golangParser) GetDependencies(flavor string, pkgPath string) []*structs.Dependency {
+	var deps []*structs.Dependency
+
+	switch flavor {
+	case packageManagerDep:
+		deps = gp.getDependenciesFromDep(pkgPath)
+	}
+
+	// Return early if no dependencies was found.
+	if len(deps) == 0 {
+		return nil
+	}
+
+	// For every dependency we should get additional data - go-import
+	// and go-source. Asynchronously.
+	var wg sync.WaitGroup
+
+	for _, dep := range deps {
+		wg.Add(1)
+		go func(dep *structs.Dependency) {
+			getGoData(dep)
+			wg.Done()
+		}(dep)
+	}
+
+	wg.Wait()
+
+	return deps
+}

parsers/parserinterface/parserinterface.go

@@ -0,0 +1,16 @@
+package parserinterface
+
+import (
+	// local
+	"go.dev.pztrn.name/glp/structs"
+)
+
+// Interface is a generic parser interface.
+type Interface interface {
+	// Detect should return true if project should be parsed using
+	// this parser and false otherwise. May optionally return package
+	// flavor (e.g. dependency management utility name).
+	Detect(pkgPath string) (bool, string)
+	// GetDependencies parses project for dependencies.
+	GetDependencies(flavor string, pkgPath string) []*structs.Dependency
+}