Initial commit.
This commit is contained in:
commit
a7bf1f29ba
|
|
@ -0,0 +1,23 @@
|
|||
# GLP
|
||||
|
||||
GLP stands for Go Licensing Processor. It is an utility that parses application's dependencies, gets their licenses and copyright information and writing gathered data into report file.
|
||||
|
||||
## Supported report file formats
|
||||
|
||||
*None yet.*
|
||||
|
||||
## Supported VCSes and sites
|
||||
|
||||
*None yet.*
|
||||
|
||||
## Installation
|
||||
|
||||
It is enough to issue:
|
||||
|
||||
```bash
|
||||
go get -u go.dev.pztrn.name/glp/cmd/glp
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
*None yet.*
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/configuration"
|
||||
"go.dev.pztrn.name/glp/httpclient"
|
||||
"go.dev.pztrn.name/glp/outputters"
|
||||
"go.dev.pztrn.name/glp/parsers"
|
||||
"go.dev.pztrn.name/glp/projecter"
|
||||
)
|
||||
|
||||
var (
|
||||
configurationPath string
|
||||
packagesPaths string
|
||||
outputFormat string
|
||||
outputFile string
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.Println("Starting glp")
|
||||
|
||||
flag.StringVar(&configurationPath, "config", "./.glp.yaml", "Path to configuration file.")
|
||||
flag.StringVar(&packagesPaths, "pkgs", "", "Packages that should be analyzed. Use comma to delimit packages.")
|
||||
flag.StringVar(&outputFormat, "outformat", "csv", "Output file format. Only 'csv' for now.")
|
||||
flag.StringVar(&outputFile, "outfile", "", "File to write licensing information to.")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
if packagesPaths == "" {
|
||||
log.Println("Packages paths that should be analyzed should be defined.")
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if outputFile == "" {
|
||||
log.Println("Output file path should be defined.")
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
configuration.Initialize(configurationPath)
|
||||
parsers.Initialize()
|
||||
outputters.Initialize()
|
||||
httpclient.Initialize()
|
||||
|
||||
projecter.Initialize(packagesPaths, outputFormat, outputFile)
|
||||
projecter.Parse()
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
package configuration
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
)
|
||||
|
||||
var (
|
||||
configurationPath string
|
||||
|
||||
Cfg *config
|
||||
)
|
||||
|
||||
// Initialize initializes package.
|
||||
func Initialize(cfgpath string) {
|
||||
log.Println("Initializing configuration")
|
||||
|
||||
configurationPath = cfgpath
|
||||
|
||||
Cfg = &config{}
|
||||
err := Cfg.initialize()
|
||||
if err != nil {
|
||||
log.Println("Error appeared when loading configuration:", err.Error())
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
package configuration
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
// This structure holds whole configuration for glp.
|
||||
type config struct {
|
||||
Log struct {
|
||||
Debug bool `yaml:"debug"`
|
||||
} `yaml:"log"`
|
||||
}
|
||||
|
||||
// Tries to parse configuration.
|
||||
func (c *config) initialize() error {
|
||||
// Check if file exists.
|
||||
if _, err := os.Stat(configurationPath); os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
// Validate configuration file path.
|
||||
// First - replace any "~" that might appear.
|
||||
if strings.Contains(configurationPath, "~") {
|
||||
userDir, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configurationPath = strings.Replace(configurationPath, "~", userDir, -1)
|
||||
}
|
||||
|
||||
// Then - make relative paths to be absolute.
|
||||
absPath, err1 := filepath.Abs(configurationPath)
|
||||
if err1 != nil {
|
||||
return err1
|
||||
}
|
||||
|
||||
configurationPath = absPath
|
||||
|
||||
log.Println("Trying to load configuration file data from '" + configurationPath + "'")
|
||||
|
||||
// Read file into memory.
|
||||
fileData, err2 := ioutil.ReadFile(configurationPath)
|
||||
if err2 != nil {
|
||||
return err2
|
||||
}
|
||||
|
||||
// Try to parse loaded data.
|
||||
err3 := yaml.Unmarshal(fileData, c)
|
||||
if err3 != nil {
|
||||
return err3
|
||||
}
|
||||
|
||||
if c.Log.Debug {
|
||||
log.Printf("Configuration parsed: %+v\n", c)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
log:
|
||||
debug: true
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
package httpclient
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/configuration"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultTimeoutInSeconds = 20
|
||||
perDomainRequestsLimit = 5
|
||||
)
|
||||
|
||||
var (
|
||||
httpClient = &http.Client{
|
||||
Timeout: time.Second * defaultTimeoutInSeconds,
|
||||
Transport: &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: time.Second * defaultTimeoutInSeconds,
|
||||
DualStack: true,
|
||||
}).DialContext,
|
||||
ExpectContinueTimeout: time.Second * 5,
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
ResponseHeaderTimeout: time.Second * defaultTimeoutInSeconds,
|
||||
TLSHandshakeTimeout: time.Second * 5,
|
||||
},
|
||||
}
|
||||
|
||||
perDomainRequests map[string]int
|
||||
perDomainRequestsMutex sync.Mutex
|
||||
)
|
||||
|
||||
// Initialize initializes package.
|
||||
func Initialize() {
|
||||
log.Println("Initializing HTTP client...")
|
||||
|
||||
perDomainRequests = make(map[string]int)
|
||||
}
|
||||
|
||||
// GET executes GET request and returns body.
|
||||
func GET(request *http.Request) []byte {
|
||||
for {
|
||||
perDomainRequestsMutex.Lock()
|
||||
currentlyRunning, found := perDomainRequests[request.URL.Host]
|
||||
perDomainRequestsMutex.Unlock()
|
||||
|
||||
if !found {
|
||||
break
|
||||
}
|
||||
|
||||
if currentlyRunning >= perDomainRequestsLimit {
|
||||
time.Sleep(time.Second * 1)
|
||||
continue
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
perDomainRequestsMutex.Lock()
|
||||
|
||||
_, found := perDomainRequests[request.URL.Host]
|
||||
if !found {
|
||||
perDomainRequests[request.URL.Host] = 1
|
||||
} else {
|
||||
perDomainRequests[request.URL.Host]++
|
||||
}
|
||||
|
||||
perDomainRequestsMutex.Unlock()
|
||||
|
||||
defer func() {
|
||||
perDomainRequestsMutex.Lock()
|
||||
|
||||
perDomainRequests[request.URL.Host]--
|
||||
|
||||
perDomainRequestsMutex.Unlock()
|
||||
}()
|
||||
|
||||
if configuration.Cfg.Log.Debug {
|
||||
log.Println("Executing request:", request.URL.String())
|
||||
}
|
||||
|
||||
var (
|
||||
requestsCount = 0
|
||||
response *http.Response
|
||||
)
|
||||
|
||||
for {
|
||||
if requestsCount == 3 {
|
||||
log.Printf("Failed to execute request %s: tried 3 times and got errors. Skipping.", request.URL.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
response, err = httpClient.Do(request)
|
||||
if err != nil {
|
||||
log.Printf("Failed to execute request %s: %s\n", request.URL.String(), err.Error())
|
||||
requestsCount++
|
||||
time.Sleep(time.Second * 1)
|
||||
continue
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
respBody, err1 := ioutil.ReadAll(response.Body)
|
||||
response.Body.Close()
|
||||
|
||||
if err1 != nil {
|
||||
log.Printf("Failed to read response body %s: %s\n", request.URL.String(), err1.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
return respBody
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
package csv
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
c "encoding/csv"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/structs"
|
||||
)
|
||||
|
||||
var (
|
||||
headers = []string{"Module", "License", "Repository URL", "License URL", "Project", "Copyrights"}
|
||||
)
|
||||
|
||||
// Responsible for pushing passed data into CSV file.
|
||||
type outputter struct{}
|
||||
|
||||
func (o *outputter) Write(deps []*structs.Dependency, outFile string) {
|
||||
log.Println("Got", strconv.Itoa(len(deps)), "dependencies to write")
|
||||
|
||||
// Check if file exists and remove it if so.
|
||||
if _, err := os.Stat(outFile); !os.IsNotExist(err) || err == nil {
|
||||
os.Remove(outFile)
|
||||
}
|
||||
|
||||
// Open file and create writer.
|
||||
f, err := os.Create(outFile)
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to open '"+outFile+"' for writing:", err.Error())
|
||||
}
|
||||
|
||||
writer := c.NewWriter(f)
|
||||
writer.Comma = ';'
|
||||
|
||||
// Write header first.
|
||||
_ = writer.Write(headers)
|
||||
|
||||
// Write dependencies information.
|
||||
for _, dep := range deps {
|
||||
_ = writer.Write([]string{dep.Name, dep.License.Name, dep.VCS.VCSPath, dep.License.URL, dep.Parent, dep.License.Copyrights})
|
||||
}
|
||||
|
||||
writer.Flush()
|
||||
|
||||
f.Close()
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package csv
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
"log"
|
||||
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/outputters/outputinterface"
|
||||
)
|
||||
|
||||
func Initialize() outputinterface.Interface {
|
||||
log.Println("Initializing csv outputter...")
|
||||
|
||||
c := &outputter{}
|
||||
return outputinterface.Interface(c)
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
package outputters
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
"log"
|
||||
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/outputters/csv"
|
||||
"go.dev.pztrn.name/glp/outputters/outputinterface"
|
||||
"go.dev.pztrn.name/glp/structs"
|
||||
)
|
||||
|
||||
var (
|
||||
outputters map[string]outputinterface.Interface
|
||||
)
|
||||
|
||||
func Initialize() {
|
||||
log.Println("Initializing output providers")
|
||||
|
||||
outputters = make(map[string]outputinterface.Interface)
|
||||
|
||||
csvIface := csv.Initialize()
|
||||
outputters["csv"] = csvIface
|
||||
}
|
||||
|
||||
// Write pushes parsed data into outputter for writing.
|
||||
func Write(outputter string, filePath string, deps []*structs.Dependency) {
|
||||
outputterIface, found := outputters[outputter]
|
||||
if !found {
|
||||
log.Fatalln("Failed to find outputter '" + outputter + "'!")
|
||||
}
|
||||
|
||||
outputterIface.Write(deps, filePath)
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
package outputinterface
|
||||
|
||||
import (
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/structs"
|
||||
)
|
||||
|
||||
// Interface is a generic output writer interface.
|
||||
type Interface interface {
|
||||
Write(deps []*structs.Dependency, outFile string)
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
package parsers
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
"errors"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/configuration"
|
||||
"go.dev.pztrn.name/glp/parsers/golang"
|
||||
"go.dev.pztrn.name/glp/parsers/parserinterface"
|
||||
"go.dev.pztrn.name/glp/structs"
|
||||
)
|
||||
|
||||
var (
|
||||
parsers map[string]parserinterface.Interface
|
||||
parsersMutex sync.RWMutex
|
||||
)
|
||||
|
||||
// Initialize initializes package.
|
||||
func Initialize() {
|
||||
log.Println("Initializing parsers...")
|
||||
|
||||
parsers = make(map[string]parserinterface.Interface)
|
||||
|
||||
// Initialize parsers.
|
||||
golangIface, golangName := golang.Initialize()
|
||||
parsers[golangName] = golangIface
|
||||
}
|
||||
|
||||
// Detect tries to launch parsers for project detection. It returns
|
||||
// parser name that should be used and optional flavor (e.g. dependencies
|
||||
// manager name) that might be returned by parser's Detect() function.
|
||||
func Detect(pkgPath string) (string, string) {
|
||||
parsersMutex.RLock()
|
||||
defer parsersMutex.RUnlock()
|
||||
|
||||
for parserName, parserIface := range parsers {
|
||||
if configuration.Cfg.Log.Debug {
|
||||
log.Println("Checking if parser '" + parserName + "' can parse project '" + pkgPath + "'...")
|
||||
}
|
||||
|
||||
useThisParser, flavor := parserIface.Detect(pkgPath)
|
||||
if useThisParser {
|
||||
return parserName, flavor
|
||||
}
|
||||
}
|
||||
|
||||
return "unknown", ""
|
||||
}
|
||||
|
||||
// GetDependencies asks parser to extract dependencies from project.
|
||||
func GetDependencies(parserName string, flavor string, pkgPath string) ([]*structs.Dependency, error) {
|
||||
parsersMutex.RLock()
|
||||
defer parsersMutex.RUnlock()
|
||||
parser, found := parsers[parserName]
|
||||
|
||||
if !found {
|
||||
return nil, errors.New("parser with such name isn't registered")
|
||||
}
|
||||
|
||||
deps := parser.GetDependencies(flavor, pkgPath)
|
||||
|
||||
return deps, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
package golang
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/configuration"
|
||||
"go.dev.pztrn.name/glp/structs"
|
||||
|
||||
// other
|
||||
"github.com/BurntSushi/toml"
|
||||
)
|
||||
|
||||
var goDepFilesToCheck = []string{"Gopkg.toml", "Gopkg.lock"}
|
||||
|
||||
type depLockConfig struct {
|
||||
Projects []struct {
|
||||
Branch string
|
||||
Digest string
|
||||
Name string
|
||||
Packages []string
|
||||
PruneOpts string
|
||||
Revision string
|
||||
Version string
|
||||
}
|
||||
SolveMeta struct {
|
||||
AnalyzerName string `toml:"analyzer-name"`
|
||||
AnalyzerVersion int `toml:"analyzer-version"`
|
||||
InputImports []string `toml:"input-imports"`
|
||||
SolverName string `toml:"solver-name"`
|
||||
SolverVersion int `toml:"solver-version"`
|
||||
} `toml:"solve-meta"`
|
||||
}
|
||||
|
||||
// Detects if project is using dep for dependencies management.
|
||||
func (gp *golangParser) detectDepUsage(pkgPath string) bool {
|
||||
var goDepFilesFound bool
|
||||
for _, fileName := range goDepFilesToCheck {
|
||||
pathToCheck := filepath.Join(pkgPath, fileName)
|
||||
if _, err := os.Stat(pathToCheck); err == nil {
|
||||
goDepFilesFound = true
|
||||
}
|
||||
}
|
||||
|
||||
if goDepFilesFound {
|
||||
log.Println("Project '" + pkgPath + "' is using dep for dependencies management")
|
||||
}
|
||||
|
||||
return goDepFilesFound
|
||||
}
|
||||
|
||||
// Gets dependencies data from dep-enabled projects.
|
||||
func (gp *golangParser) getDependenciesFromDep(pkgPath string) []*structs.Dependency {
|
||||
deps := make([]*structs.Dependency, 0)
|
||||
|
||||
// Try to figure out parent package name for all dependencies.
|
||||
parent := gp.getParentForDep(pkgPath)
|
||||
|
||||
// All dependencies for project will be taken from Gopkg.lock file.
|
||||
lockFile := &depLockConfig{}
|
||||
_, err := toml.DecodeFile(filepath.Join(pkgPath, "Gopkg.lock"), lockFile)
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to parse dep lock file:", err.Error())
|
||||
}
|
||||
|
||||
if configuration.Cfg.Log.Debug {
|
||||
log.Printf("dep lock file parsed: %+v\n", lockFile)
|
||||
}
|
||||
|
||||
// Parse dependencies.
|
||||
for _, dep := range lockFile.Projects {
|
||||
dependency := &structs.Dependency{
|
||||
Name: dep.Name,
|
||||
Parent: parent,
|
||||
VCS: structs.VCSData{
|
||||
Branch: dep.Branch,
|
||||
Revision: dep.Revision,
|
||||
},
|
||||
Version: dep.Version,
|
||||
}
|
||||
|
||||
// If branch is empty - assume master.
|
||||
if dependency.VCS.Branch == "" {
|
||||
dependency.VCS.Branch = "master"
|
||||
}
|
||||
|
||||
deps = append(deps, dependency)
|
||||
|
||||
if configuration.Cfg.Log.Debug {
|
||||
log.Printf("Initial dependency structure formed: %+v\n", dependency)
|
||||
}
|
||||
}
|
||||
|
||||
return deps
|
||||
}
|
||||
|
||||
// Tries to get package name for passed package path.
|
||||
func (gp *golangParser) getParentForDep(pkgPath string) string {
|
||||
// Dep-managed projects are in 99% of cases are placed in GOPATH.
|
||||
if strings.Contains(pkgPath, "src") {
|
||||
return strings.Split(pkgPath, "src/")[1]
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package golang
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
"log"
|
||||
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/parsers/parserinterface"
|
||||
)
|
||||
|
||||
func Initialize() (parserinterface.Interface, string) {
|
||||
log.Println("Initializing Golang projects parser")
|
||||
|
||||
p := &golangParser{}
|
||||
return parserinterface.Interface(p), "golang"
|
||||
}
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
package golang
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
"bytes"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/configuration"
|
||||
"go.dev.pztrn.name/glp/httpclient"
|
||||
"go.dev.pztrn.name/glp/structs"
|
||||
)
|
||||
|
||||
// Gets go-import and go-source data and fill it in dependency.
|
||||
func getGoData(dependency *structs.Dependency) {
|
||||
// Dependencies are imported using URL which can be called with
|
||||
// "?go-get=1" parameter to obtain required VCS data.
|
||||
req, _ := http.NewRequest("GET", "http://"+dependency.Name, nil)
|
||||
|
||||
q := req.URL.Query()
|
||||
q.Add("go-get", "1")
|
||||
|
||||
req.URL.RawQuery = q.Encode()
|
||||
|
||||
respBody := httpclient.GET(req)
|
||||
if respBody == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// HTML is hard to parse properly statically, so we will go
|
||||
// line-by-line for <head> parsing.
|
||||
resp := bytes.NewBuffer(respBody)
|
||||
|
||||
var (
|
||||
// This flag shows that we're currently parsing <head> from HTML.
|
||||
headCurrentlyParsing bool
|
||||
)
|
||||
|
||||
for {
|
||||
line, err := resp.ReadString('\n')
|
||||
if err != nil && err != io.EOF {
|
||||
log.Fatalln("Failed to read HTML response line-by-line:", err.Error())
|
||||
} else if err != nil && err == io.EOF {
|
||||
break
|
||||
}
|
||||
|
||||
if headCurrentlyParsing {
|
||||
// Check for go-import data.
|
||||
if strings.Contains(line, `<meta name="go-import"`) {
|
||||
// Get content.
|
||||
// Import things are in element #4.
|
||||
lineSplitted := strings.Split(line, `"`)
|
||||
|
||||
// Check line length. This is not so good approach, but
|
||||
// should work for 99% of dependencies.
|
||||
if len(lineSplitted) < 5 {
|
||||
log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - tag itself appears to be incomplete. Skipping")
|
||||
continue
|
||||
}
|
||||
|
||||
if len(lineSplitted) > 5 {
|
||||
log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - line where meta tag is located appears to be too long. Skipping")
|
||||
continue
|
||||
}
|
||||
|
||||
// Import line contains data like VCS name and VCS URL.
|
||||
// They're delimited with whitespace.
|
||||
importDataSplitted := strings.Split(lineSplitted[3], " ")
|
||||
|
||||
// Import line should contain at least 3 elements.
|
||||
if len(importDataSplitted) < 3 {
|
||||
log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - import data is too small. Skipping")
|
||||
continue
|
||||
}
|
||||
|
||||
// Fill dependency data with this data.
|
||||
// First element is a module name and we do not actually
|
||||
// need it, because it is already filled previously.
|
||||
dependency.VCS.VCS = importDataSplitted[1]
|
||||
dependency.VCS.VCSPath = importDataSplitted[2]
|
||||
}
|
||||
|
||||
// Check for go-source data.
|
||||
if strings.Contains(line, `<meta name="go-source"`) {
|
||||
// Get content.
|
||||
// Import things are in element #4.
|
||||
lineSplitted := strings.Split(line, `"`)
|
||||
|
||||
// Check line length. This is not so good approach, but
|
||||
// should work for 99% of dependencies.
|
||||
if len(lineSplitted) < 5 {
|
||||
log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - tag itself appears to be incomplete. Skipping")
|
||||
continue
|
||||
}
|
||||
|
||||
if len(lineSplitted) > 5 {
|
||||
log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - line where meta tag is located appears to be too long. Skipping")
|
||||
continue
|
||||
}
|
||||
|
||||
// Source line contains data like VCS paths templates.
|
||||
// They're delimited with whitespace.
|
||||
sourceDataSplitted := strings.Split(lineSplitted[3], " ")
|
||||
|
||||
// Source data line should contain at least 3 elements.
|
||||
if len(sourceDataSplitted) < 4 {
|
||||
log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - source data is too small. Skipping")
|
||||
continue
|
||||
}
|
||||
|
||||
// Fill dependency data.
|
||||
dependency.VCS.SourceURLDirTemplate = sourceDataSplitted[2]
|
||||
dependency.VCS.SourceURLFileTemplate = sourceDataSplitted[3]
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Contains(strings.ToLower(line), "<head>") {
|
||||
headCurrentlyParsing = true
|
||||
}
|
||||
|
||||
if strings.Contains(strings.ToLower(line), "</head>") {
|
||||
headCurrentlyParsing = false
|
||||
}
|
||||
}
|
||||
|
||||
if configuration.Cfg.Log.Debug {
|
||||
log.Printf("go-import and go-source data parsed: %+v\n", dependency.VCS)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
package golang
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
var goModulesFilesToCheck = []string{"go.mod", "go.sum"}
|
||||
|
||||
// Detects if project is using go modules for dependencies management.
|
||||
func (gp *golangParser) detectModulesUsage(pkgPath string) bool {
|
||||
var goModulesFileFound bool
|
||||
for _, fileName := range goModulesFilesToCheck {
|
||||
pathToCheck := filepath.Join(pkgPath, fileName)
|
||||
if _, err := os.Stat(pathToCheck); err == nil {
|
||||
goModulesFileFound = true
|
||||
}
|
||||
}
|
||||
|
||||
if goModulesFileFound {
|
||||
log.Println("Project '" + pkgPath + "' is using Go modules for dependencies management")
|
||||
}
|
||||
|
||||
return goModulesFileFound
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
package golang
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
"sync"
|
||||
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/structs"
|
||||
)
|
||||
|
||||
const (
|
||||
// Package managers names. Used in Detect() for flavor returning.
|
||||
packageManagerGoMod = "go mod"
|
||||
packageManagerDep = "dep"
|
||||
)
|
||||
|
||||
// This structure responsible for parsing projects that written in Go.
|
||||
type golangParser struct{}
|
||||
|
||||
// Detect detects if passed project path can be parsed with this parser
|
||||
// and additionally detect package manager used.
|
||||
func (gp *golangParser) Detect(pkgPath string) (bool, string) {
|
||||
// Go projects usually using go modules or dep for dependencies
|
||||
// management.
|
||||
isModules := gp.detectModulesUsage(pkgPath)
|
||||
if isModules {
|
||||
return true, packageManagerGoMod
|
||||
}
|
||||
|
||||
isDep := gp.detectDepUsage(pkgPath)
|
||||
if isDep {
|
||||
return true, packageManagerDep
|
||||
}
|
||||
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// GetDependencies extracts dependencies from project.
|
||||
func (gp *golangParser) GetDependencies(flavor string, pkgPath string) []*structs.Dependency {
|
||||
var deps []*structs.Dependency
|
||||
|
||||
switch flavor {
|
||||
case packageManagerDep:
|
||||
deps = gp.getDependenciesFromDep(pkgPath)
|
||||
}
|
||||
|
||||
// Return early if no dependencies was found.
|
||||
if len(deps) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// For every dependency we should get additional data - go-import
|
||||
// and go-source. Asynchronously.
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, dep := range deps {
|
||||
wg.Add(1)
|
||||
go func(dep *structs.Dependency) {
|
||||
getGoData(dep)
|
||||
wg.Done()
|
||||
}(dep)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
return deps
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package parserinterface
|
||||
|
||||
import (
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/structs"
|
||||
)
|
||||
|
||||
// Interface is a generic parser interface.
|
||||
type Interface interface {
|
||||
// Detect should return true if project should be parsed using
|
||||
// this parser and false otherwise. May optionally return package
|
||||
// flavor (e.g. dependency management utility name).
|
||||
Detect(pkgPath string) (bool, string)
|
||||
// GetDependencies parses project for dependencies.
|
||||
GetDependencies(flavor string, pkgPath string) []*structs.Dependency
|
||||
}
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
package projecter
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/configuration"
|
||||
"go.dev.pztrn.name/glp/outputters"
|
||||
"go.dev.pztrn.name/glp/structs"
|
||||
)
|
||||
|
||||
var (
|
||||
packages []string
|
||||
|
||||
outputFormat string
|
||||
outputFile string
|
||||
|
||||
projects map[string]*Project
|
||||
projectsMutex sync.RWMutex
|
||||
)
|
||||
|
||||
// Initialize initializes package.
|
||||
func Initialize(pkgs string, outFormat string, outFile string) {
|
||||
log.Println("Initializing projects handler...")
|
||||
|
||||
packages = strings.Split(pkgs, ",")
|
||||
projects = make(map[string]*Project)
|
||||
|
||||
outputFormat = outFormat
|
||||
outputFile = outFile
|
||||
|
||||
log.Println("Packages list that was passed:", packages)
|
||||
}
|
||||
|
||||
// GetProject returns project by it's path.
|
||||
func GetProject(path string) *Project {
|
||||
projectsMutex.RLock()
|
||||
defer projectsMutex.RUnlock()
|
||||
|
||||
prj, found := projects[path]
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
|
||||
return prj
|
||||
}
|
||||
|
||||
// Parse starts projects parsing.
|
||||
func Parse() {
|
||||
// Create project for every passed package.
|
||||
// This is done in main goroutine and therefore no mutex is used.
|
||||
for _, pkgPath := range packages {
|
||||
prj := NewProject(pkgPath)
|
||||
projects[pkgPath] = prj
|
||||
}
|
||||
|
||||
if configuration.Cfg.Log.Debug {
|
||||
log.Printf("Projects generated: %+v\n", projects)
|
||||
}
|
||||
|
||||
// We should start asynchronous projects parsing.
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, prj := range projects {
|
||||
wg.Add(1)
|
||||
go func(prj *Project) {
|
||||
prj.process()
|
||||
wg.Done()
|
||||
}(prj)
|
||||
}
|
||||
|
||||
// Wait until all projects will be parsed.
|
||||
wg.Wait()
|
||||
|
||||
// Collect dependencies list from all parsed projects.
|
||||
var deps []*structs.Dependency
|
||||
|
||||
for _, prj := range projects {
|
||||
deps = append(deps, prj.GetDeps()...)
|
||||
}
|
||||
|
||||
outputters.Write(outputFormat, outputFile, deps)
|
||||
|
||||
log.Println("Parsing done")
|
||||
}
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
package projecter
|
||||
|
||||
import (
|
||||
// stdlib
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
// local
|
||||
"go.dev.pztrn.name/glp/parsers"
|
||||
"go.dev.pztrn.name/glp/structs"
|
||||
)
|
||||
|
||||
// Project represents single project (or package) that was passed via
|
||||
// -pkgs parameter.
|
||||
type Project struct {
|
||||
packagePath string
|
||||
parserName string
|
||||
flavor string
|
||||
|
||||
deps []*structs.Dependency
|
||||
}
|
||||
|
||||
// NewProject creates new project and returns it.
|
||||
func NewProject(packagePath string) *Project {
|
||||
p := &Project{}
|
||||
p.initialize(packagePath)
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
// GetDeps returns list of dependencies for project.
|
||||
func (p *Project) GetDeps() []*structs.Dependency {
|
||||
return p.deps
|
||||
}
|
||||
|
||||
// Initializes project.
|
||||
func (p *Project) initialize(packagePath string) {
|
||||
p.packagePath = packagePath
|
||||
|
||||
// Prepare package path to be used.
|
||||
// First - replace "~" with actual home directory.
|
||||
if strings.Contains(p.packagePath, "~") {
|
||||
homeDir, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to get user's home directory:", err.Error())
|
||||
}
|
||||
|
||||
p.packagePath = strings.Replace(p.packagePath, "~", homeDir, -1)
|
||||
}
|
||||
|
||||
// Get absolute path.
|
||||
var err error
|
||||
p.packagePath, err = filepath.Abs(p.packagePath)
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to get absolute path for package '"+p.packagePath+":", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// Starts project parsing.
|
||||
func (p *Project) process() {
|
||||
// We should determine project type.
|
||||
p.parserName, p.flavor = parsers.Detect(p.packagePath)
|
||||
|
||||
if p.parserName == "unknown" {
|
||||
log.Println("Project", p.packagePath, "cannot be parsed with glp")
|
||||
return
|
||||
}
|
||||
|
||||
// Lets try to get dependencies, their versions and URLs.
|
||||
deps, err := parsers.GetDependencies(p.parserName, p.flavor, p.packagePath)
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to get dependencies:", err.Error())
|
||||
}
|
||||
|
||||
p.deps = deps
|
||||
|
||||
// ToDo: get licensing information.
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
package structs
|
||||
|
||||
// Dependency represents single dependency data.
|
||||
type Dependency struct {
|
||||
// License is a license name for dependency.
|
||||
License License
|
||||
// Name is a dependency name as it appears in package manager's
|
||||
// lock file or in sources if no package manager is used.
|
||||
Name string
|
||||
// Parent is a path to parent package.
|
||||
Parent string
|
||||
// VCS is a VCS data obtained for dependency.
|
||||
VCS VCSData
|
||||
// Version is a dependency version used in project.
|
||||
Version string
|
||||
// URL is a web URL for that dependency (Github, Gitlab, etc.).
|
||||
URL string
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package structs
|
||||
|
||||
type License struct {
|
||||
Copyrights string
|
||||
Name string
|
||||
URL string
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package structs
|
||||
|
||||
// VCSData describes structure of go-import and go-source data.
|
||||
type VCSData struct {
|
||||
// Branch is a VCS branch used.
|
||||
Branch string
|
||||
// Revision is a VCS revision used.
|
||||
Revision string
|
||||
// SourceURLDirTemplate is a template for sources dirs URLs. E.g.:
|
||||
// https://sources.dev.pztrn.name/pztrn/glp/src/branch/master{/dir}
|
||||
SourceURLDirTemplate string
|
||||
// SourceURLFileTemplate is a template for sources files URLs. E.g.:
|
||||
// https://sources.dev.pztrn.name/pztrn/glp/src/branch/master{/dir}/{file}#L{line}
|
||||
SourceURLFileTemplate string
|
||||
// VCS is a VCS name (e.g. "git").
|
||||
VCS string
|
||||
// VCSPath is a VCS repository path.
|
||||
VCSPath string
|
||||
}
|
||||
Loading…
Reference in New Issue