From a7bf1f29baa986323436ecd8be6b749ce80fbab7 Mon Sep 17 00:00:00 2001 From: "Stanislav N. aka pztrn" Date: Thu, 6 Feb 2020 15:09:25 +0500 Subject: [PATCH] Initial commit. --- README.md | 23 +++ cmd/glp/main.go | 53 +++++++ configuration/exported.go | 29 ++++ configuration/struct.go | 65 +++++++++ glp.example.yaml | 2 + httpclient/exported.go | 122 ++++++++++++++++ outputters/csv/csv.go | 49 +++++++ outputters/csv/exported.go | 16 +++ outputters/exported.go | 34 +++++ outputters/outputinterface/outputinterface.go | 11 ++ parsers/exported.go | 66 +++++++++ parsers/golang/dep.go | 109 +++++++++++++++ parsers/golang/exported.go | 16 +++ parsers/golang/godata.go | 132 ++++++++++++++++++ parsers/golang/modules.go | 27 ++++ parsers/golang/parser.go | 67 +++++++++ parsers/parserinterface/parserinterface.go | 16 +++ projecter/exported.go | 88 ++++++++++++ projecter/project.go | 80 +++++++++++ structs/dependency.go | 18 +++ structs/license.go | 7 + structs/vcsdata.go | 19 +++ 22 files changed, 1049 insertions(+) create mode 100644 README.md create mode 100644 cmd/glp/main.go create mode 100644 configuration/exported.go create mode 100644 configuration/struct.go create mode 100644 glp.example.yaml create mode 100644 httpclient/exported.go create mode 100644 outputters/csv/csv.go create mode 100644 outputters/csv/exported.go create mode 100644 outputters/exported.go create mode 100644 outputters/outputinterface/outputinterface.go create mode 100644 parsers/exported.go create mode 100644 parsers/golang/dep.go create mode 100644 parsers/golang/exported.go create mode 100644 parsers/golang/godata.go create mode 100644 parsers/golang/modules.go create mode 100644 parsers/golang/parser.go create mode 100644 parsers/parserinterface/parserinterface.go create mode 100644 projecter/exported.go create mode 100644 projecter/project.go create mode 100644 structs/dependency.go create mode 100644 structs/license.go create mode 100644 structs/vcsdata.go diff --git a/README.md b/README.md new file mode 100644 index 0000000..69d2ef5 --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# GLP + +GLP stands for Go Licensing Processor. It is an utility that parses application's dependencies, gets their licenses and copyright information and writing gathered data into report file. + +## Supported report file formats + +*None yet.* + +## Supported VCSes and sites + +*None yet.* + +## Installation + +It is enough to issue: + +```bash +go get -u go.dev.pztrn.name/glp/cmd/glp +``` + +## Configuration + +*None yet.* diff --git a/cmd/glp/main.go b/cmd/glp/main.go new file mode 100644 index 0000000..8b40f36 --- /dev/null +++ b/cmd/glp/main.go @@ -0,0 +1,53 @@ +package main + +import ( + // stdlib + "flag" + "log" + "os" + + // local + "go.dev.pztrn.name/glp/configuration" + "go.dev.pztrn.name/glp/httpclient" + "go.dev.pztrn.name/glp/outputters" + "go.dev.pztrn.name/glp/parsers" + "go.dev.pztrn.name/glp/projecter" +) + +var ( + configurationPath string + packagesPaths string + outputFormat string + outputFile string +) + +func main() { + log.Println("Starting glp") + + flag.StringVar(&configurationPath, "config", "./.glp.yaml", "Path to configuration file.") + flag.StringVar(&packagesPaths, "pkgs", "", "Packages that should be analyzed. Use comma to delimit packages.") + flag.StringVar(&outputFormat, "outformat", "csv", "Output file format. Only 'csv' for now.") + flag.StringVar(&outputFile, "outfile", "", "File to write licensing information to.") + + flag.Parse() + + if packagesPaths == "" { + log.Println("Packages paths that should be analyzed should be defined.") + flag.PrintDefaults() + os.Exit(1) + } + + if outputFile == "" { + log.Println("Output file path should be defined.") + flag.PrintDefaults() + os.Exit(1) + } + + configuration.Initialize(configurationPath) + parsers.Initialize() + outputters.Initialize() + httpclient.Initialize() + + projecter.Initialize(packagesPaths, outputFormat, outputFile) + projecter.Parse() +} diff --git a/configuration/exported.go b/configuration/exported.go new file mode 100644 index 0000000..b59ae72 --- /dev/null +++ b/configuration/exported.go @@ -0,0 +1,29 @@ +package configuration + +import ( + // stdlib + "flag" + "log" + "os" +) + +var ( + configurationPath string + + Cfg *config +) + +// Initialize initializes package. +func Initialize(cfgpath string) { + log.Println("Initializing configuration") + + configurationPath = cfgpath + + Cfg = &config{} + err := Cfg.initialize() + if err != nil { + log.Println("Error appeared when loading configuration:", err.Error()) + flag.PrintDefaults() + os.Exit(1) + } +} diff --git a/configuration/struct.go b/configuration/struct.go new file mode 100644 index 0000000..ac4be85 --- /dev/null +++ b/configuration/struct.go @@ -0,0 +1,65 @@ +package configuration + +import ( + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v2" +) + +// This structure holds whole configuration for glp. +type config struct { + Log struct { + Debug bool `yaml:"debug"` + } `yaml:"log"` +} + +// Tries to parse configuration. +func (c *config) initialize() error { + // Check if file exists. + if _, err := os.Stat(configurationPath); os.IsNotExist(err) { + return err + } + + // Validate configuration file path. + // First - replace any "~" that might appear. + if strings.Contains(configurationPath, "~") { + userDir, err := os.UserHomeDir() + if err != nil { + return err + } + + configurationPath = strings.Replace(configurationPath, "~", userDir, -1) + } + + // Then - make relative paths to be absolute. + absPath, err1 := filepath.Abs(configurationPath) + if err1 != nil { + return err1 + } + + configurationPath = absPath + + log.Println("Trying to load configuration file data from '" + configurationPath + "'") + + // Read file into memory. + fileData, err2 := ioutil.ReadFile(configurationPath) + if err2 != nil { + return err2 + } + + // Try to parse loaded data. + err3 := yaml.Unmarshal(fileData, c) + if err3 != nil { + return err3 + } + + if c.Log.Debug { + log.Printf("Configuration parsed: %+v\n", c) + } + + return nil +} diff --git a/glp.example.yaml b/glp.example.yaml new file mode 100644 index 0000000..0e7a35d --- /dev/null +++ b/glp.example.yaml @@ -0,0 +1,2 @@ +log: + debug: true diff --git a/httpclient/exported.go b/httpclient/exported.go new file mode 100644 index 0000000..af99f40 --- /dev/null +++ b/httpclient/exported.go @@ -0,0 +1,122 @@ +package httpclient + +import ( + // stdlib + "io/ioutil" + "log" + "net" + "net/http" + "sync" + "time" + + // local + "go.dev.pztrn.name/glp/configuration" +) + +const ( + defaultTimeoutInSeconds = 20 + perDomainRequestsLimit = 5 +) + +var ( + httpClient = &http.Client{ + Timeout: time.Second * defaultTimeoutInSeconds, + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: time.Second * defaultTimeoutInSeconds, + DualStack: true, + }).DialContext, + ExpectContinueTimeout: time.Second * 5, + Proxy: http.ProxyFromEnvironment, + ResponseHeaderTimeout: time.Second * defaultTimeoutInSeconds, + TLSHandshakeTimeout: time.Second * 5, + }, + } + + perDomainRequests map[string]int + perDomainRequestsMutex sync.Mutex +) + +// Initialize initializes package. +func Initialize() { + log.Println("Initializing HTTP client...") + + perDomainRequests = make(map[string]int) +} + +// GET executes GET request and returns body. +func GET(request *http.Request) []byte { + for { + perDomainRequestsMutex.Lock() + currentlyRunning, found := perDomainRequests[request.URL.Host] + perDomainRequestsMutex.Unlock() + + if !found { + break + } + + if currentlyRunning >= perDomainRequestsLimit { + time.Sleep(time.Second * 1) + continue + } + + break + } + + perDomainRequestsMutex.Lock() + + _, found := perDomainRequests[request.URL.Host] + if !found { + perDomainRequests[request.URL.Host] = 1 + } else { + perDomainRequests[request.URL.Host]++ + } + + perDomainRequestsMutex.Unlock() + + defer func() { + perDomainRequestsMutex.Lock() + + perDomainRequests[request.URL.Host]-- + + perDomainRequestsMutex.Unlock() + }() + + if configuration.Cfg.Log.Debug { + log.Println("Executing request:", request.URL.String()) + } + + var ( + requestsCount = 0 + response *http.Response + ) + + for { + if requestsCount == 3 { + log.Printf("Failed to execute request %s: tried 3 times and got errors. Skipping.", request.URL.String()) + return nil + } + + var err error + + response, err = httpClient.Do(request) + if err != nil { + log.Printf("Failed to execute request %s: %s\n", request.URL.String(), err.Error()) + requestsCount++ + time.Sleep(time.Second * 1) + continue + } + + break + } + + respBody, err1 := ioutil.ReadAll(response.Body) + response.Body.Close() + + if err1 != nil { + log.Printf("Failed to read response body %s: %s\n", request.URL.String(), err1.Error()) + return nil + } + + return respBody +} diff --git a/outputters/csv/csv.go b/outputters/csv/csv.go new file mode 100644 index 0000000..779b373 --- /dev/null +++ b/outputters/csv/csv.go @@ -0,0 +1,49 @@ +package csv + +import ( + // stdlib + c "encoding/csv" + "log" + "os" + "strconv" + + // local + "go.dev.pztrn.name/glp/structs" +) + +var ( + headers = []string{"Module", "License", "Repository URL", "License URL", "Project", "Copyrights"} +) + +// Responsible for pushing passed data into CSV file. +type outputter struct{} + +func (o *outputter) Write(deps []*structs.Dependency, outFile string) { + log.Println("Got", strconv.Itoa(len(deps)), "dependencies to write") + + // Check if file exists and remove it if so. + if _, err := os.Stat(outFile); !os.IsNotExist(err) || err == nil { + os.Remove(outFile) + } + + // Open file and create writer. + f, err := os.Create(outFile) + if err != nil { + log.Fatalln("Failed to open '"+outFile+"' for writing:", err.Error()) + } + + writer := c.NewWriter(f) + writer.Comma = ';' + + // Write header first. + _ = writer.Write(headers) + + // Write dependencies information. + for _, dep := range deps { + _ = writer.Write([]string{dep.Name, dep.License.Name, dep.VCS.VCSPath, dep.License.URL, dep.Parent, dep.License.Copyrights}) + } + + writer.Flush() + + f.Close() +} diff --git a/outputters/csv/exported.go b/outputters/csv/exported.go new file mode 100644 index 0000000..3708015 --- /dev/null +++ b/outputters/csv/exported.go @@ -0,0 +1,16 @@ +package csv + +import ( + // stdlib + "log" + + // local + "go.dev.pztrn.name/glp/outputters/outputinterface" +) + +func Initialize() outputinterface.Interface { + log.Println("Initializing csv outputter...") + + c := &outputter{} + return outputinterface.Interface(c) +} diff --git a/outputters/exported.go b/outputters/exported.go new file mode 100644 index 0000000..6a70c6d --- /dev/null +++ b/outputters/exported.go @@ -0,0 +1,34 @@ +package outputters + +import ( + // stdlib + "log" + + // local + "go.dev.pztrn.name/glp/outputters/csv" + "go.dev.pztrn.name/glp/outputters/outputinterface" + "go.dev.pztrn.name/glp/structs" +) + +var ( + outputters map[string]outputinterface.Interface +) + +func Initialize() { + log.Println("Initializing output providers") + + outputters = make(map[string]outputinterface.Interface) + + csvIface := csv.Initialize() + outputters["csv"] = csvIface +} + +// Write pushes parsed data into outputter for writing. +func Write(outputter string, filePath string, deps []*structs.Dependency) { + outputterIface, found := outputters[outputter] + if !found { + log.Fatalln("Failed to find outputter '" + outputter + "'!") + } + + outputterIface.Write(deps, filePath) +} diff --git a/outputters/outputinterface/outputinterface.go b/outputters/outputinterface/outputinterface.go new file mode 100644 index 0000000..eeadfb4 --- /dev/null +++ b/outputters/outputinterface/outputinterface.go @@ -0,0 +1,11 @@ +package outputinterface + +import ( + // local + "go.dev.pztrn.name/glp/structs" +) + +// Interface is a generic output writer interface. +type Interface interface { + Write(deps []*structs.Dependency, outFile string) +} diff --git a/parsers/exported.go b/parsers/exported.go new file mode 100644 index 0000000..5d76acb --- /dev/null +++ b/parsers/exported.go @@ -0,0 +1,66 @@ +package parsers + +import ( + // stdlib + "errors" + "log" + "sync" + + // local + "go.dev.pztrn.name/glp/configuration" + "go.dev.pztrn.name/glp/parsers/golang" + "go.dev.pztrn.name/glp/parsers/parserinterface" + "go.dev.pztrn.name/glp/structs" +) + +var ( + parsers map[string]parserinterface.Interface + parsersMutex sync.RWMutex +) + +// Initialize initializes package. +func Initialize() { + log.Println("Initializing parsers...") + + parsers = make(map[string]parserinterface.Interface) + + // Initialize parsers. + golangIface, golangName := golang.Initialize() + parsers[golangName] = golangIface +} + +// Detect tries to launch parsers for project detection. It returns +// parser name that should be used and optional flavor (e.g. dependencies +// manager name) that might be returned by parser's Detect() function. +func Detect(pkgPath string) (string, string) { + parsersMutex.RLock() + defer parsersMutex.RUnlock() + + for parserName, parserIface := range parsers { + if configuration.Cfg.Log.Debug { + log.Println("Checking if parser '" + parserName + "' can parse project '" + pkgPath + "'...") + } + + useThisParser, flavor := parserIface.Detect(pkgPath) + if useThisParser { + return parserName, flavor + } + } + + return "unknown", "" +} + +// GetDependencies asks parser to extract dependencies from project. +func GetDependencies(parserName string, flavor string, pkgPath string) ([]*structs.Dependency, error) { + parsersMutex.RLock() + defer parsersMutex.RUnlock() + parser, found := parsers[parserName] + + if !found { + return nil, errors.New("parser with such name isn't registered") + } + + deps := parser.GetDependencies(flavor, pkgPath) + + return deps, nil +} diff --git a/parsers/golang/dep.go b/parsers/golang/dep.go new file mode 100644 index 0000000..92b6f15 --- /dev/null +++ b/parsers/golang/dep.go @@ -0,0 +1,109 @@ +package golang + +import ( + // stdlib + "log" + "os" + "path/filepath" + "strings" + + // local + "go.dev.pztrn.name/glp/configuration" + "go.dev.pztrn.name/glp/structs" + + // other + "github.com/BurntSushi/toml" +) + +var goDepFilesToCheck = []string{"Gopkg.toml", "Gopkg.lock"} + +type depLockConfig struct { + Projects []struct { + Branch string + Digest string + Name string + Packages []string + PruneOpts string + Revision string + Version string + } + SolveMeta struct { + AnalyzerName string `toml:"analyzer-name"` + AnalyzerVersion int `toml:"analyzer-version"` + InputImports []string `toml:"input-imports"` + SolverName string `toml:"solver-name"` + SolverVersion int `toml:"solver-version"` + } `toml:"solve-meta"` +} + +// Detects if project is using dep for dependencies management. +func (gp *golangParser) detectDepUsage(pkgPath string) bool { + var goDepFilesFound bool + for _, fileName := range goDepFilesToCheck { + pathToCheck := filepath.Join(pkgPath, fileName) + if _, err := os.Stat(pathToCheck); err == nil { + goDepFilesFound = true + } + } + + if goDepFilesFound { + log.Println("Project '" + pkgPath + "' is using dep for dependencies management") + } + + return goDepFilesFound +} + +// Gets dependencies data from dep-enabled projects. +func (gp *golangParser) getDependenciesFromDep(pkgPath string) []*structs.Dependency { + deps := make([]*structs.Dependency, 0) + + // Try to figure out parent package name for all dependencies. + parent := gp.getParentForDep(pkgPath) + + // All dependencies for project will be taken from Gopkg.lock file. + lockFile := &depLockConfig{} + _, err := toml.DecodeFile(filepath.Join(pkgPath, "Gopkg.lock"), lockFile) + if err != nil { + log.Fatalln("Failed to parse dep lock file:", err.Error()) + } + + if configuration.Cfg.Log.Debug { + log.Printf("dep lock file parsed: %+v\n", lockFile) + } + + // Parse dependencies. + for _, dep := range lockFile.Projects { + dependency := &structs.Dependency{ + Name: dep.Name, + Parent: parent, + VCS: structs.VCSData{ + Branch: dep.Branch, + Revision: dep.Revision, + }, + Version: dep.Version, + } + + // If branch is empty - assume master. + if dependency.VCS.Branch == "" { + dependency.VCS.Branch = "master" + } + + deps = append(deps, dependency) + + if configuration.Cfg.Log.Debug { + log.Printf("Initial dependency structure formed: %+v\n", dependency) + } + } + + return deps +} + +// Tries to get package name for passed package path. +func (gp *golangParser) getParentForDep(pkgPath string) string { + // Dep-managed projects are in 99% of cases are placed in GOPATH. + if strings.Contains(pkgPath, "src") { + return strings.Split(pkgPath, "src/")[1] + } + + return "" +} diff --git a/parsers/golang/exported.go b/parsers/golang/exported.go new file mode 100644 index 0000000..79c54c5 --- /dev/null +++ b/parsers/golang/exported.go @@ -0,0 +1,16 @@ +package golang + +import ( + // stdlib + "log" + + // local + "go.dev.pztrn.name/glp/parsers/parserinterface" +) + +func Initialize() (parserinterface.Interface, string) { + log.Println("Initializing Golang projects parser") + + p := &golangParser{} + return parserinterface.Interface(p), "golang" +} diff --git a/parsers/golang/godata.go b/parsers/golang/godata.go new file mode 100644 index 0000000..79878f3 --- /dev/null +++ b/parsers/golang/godata.go @@ -0,0 +1,132 @@ +package golang + +import ( + // stdlib + "bytes" + "io" + "log" + "net/http" + "strings" + + // local + "go.dev.pztrn.name/glp/configuration" + "go.dev.pztrn.name/glp/httpclient" + "go.dev.pztrn.name/glp/structs" +) + +// Gets go-import and go-source data and fill it in dependency. +func getGoData(dependency *structs.Dependency) { + // Dependencies are imported using URL which can be called with + // "?go-get=1" parameter to obtain required VCS data. + req, _ := http.NewRequest("GET", "http://"+dependency.Name, nil) + + q := req.URL.Query() + q.Add("go-get", "1") + + req.URL.RawQuery = q.Encode() + + respBody := httpclient.GET(req) + if respBody == nil { + return + } + + // HTML is hard to parse properly statically, so we will go + // line-by-line for parsing. + resp := bytes.NewBuffer(respBody) + + var ( + // This flag shows that we're currently parsing from HTML. + headCurrentlyParsing bool + ) + + for { + line, err := resp.ReadString('\n') + if err != nil && err != io.EOF { + log.Fatalln("Failed to read HTML response line-by-line:", err.Error()) + } else if err != nil && err == io.EOF { + break + } + + if headCurrentlyParsing { + // Check for go-import data. + if strings.Contains(line, ` 5 { + log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - line where meta tag is located appears to be too long. Skipping") + continue + } + + // Import line contains data like VCS name and VCS URL. + // They're delimited with whitespace. + importDataSplitted := strings.Split(lineSplitted[3], " ") + + // Import line should contain at least 3 elements. + if len(importDataSplitted) < 3 { + log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - import data is too small. Skipping") + continue + } + + // Fill dependency data with this data. + // First element is a module name and we do not actually + // need it, because it is already filled previously. + dependency.VCS.VCS = importDataSplitted[1] + dependency.VCS.VCSPath = importDataSplitted[2] + } + + // Check for go-source data. + if strings.Contains(line, ` 5 { + log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - line where meta tag is located appears to be too long. Skipping") + continue + } + + // Source line contains data like VCS paths templates. + // They're delimited with whitespace. + sourceDataSplitted := strings.Split(lineSplitted[3], " ") + + // Source data line should contain at least 3 elements. + if len(sourceDataSplitted) < 4 { + log.Println("Got line: '" + line + "', but it cannot be parsed. Probably badly formed - source data is too small. Skipping") + continue + } + + // Fill dependency data. + dependency.VCS.SourceURLDirTemplate = sourceDataSplitted[2] + dependency.VCS.SourceURLFileTemplate = sourceDataSplitted[3] + } + } + + if strings.Contains(strings.ToLower(line), "") { + headCurrentlyParsing = true + } + + if strings.Contains(strings.ToLower(line), "") { + headCurrentlyParsing = false + } + } + + if configuration.Cfg.Log.Debug { + log.Printf("go-import and go-source data parsed: %+v\n", dependency.VCS) + } +} diff --git a/parsers/golang/modules.go b/parsers/golang/modules.go new file mode 100644 index 0000000..b5d4012 --- /dev/null +++ b/parsers/golang/modules.go @@ -0,0 +1,27 @@ +package golang + +import ( + // stdlib + "log" + "os" + "path/filepath" +) + +var goModulesFilesToCheck = []string{"go.mod", "go.sum"} + +// Detects if project is using go modules for dependencies management. +func (gp *golangParser) detectModulesUsage(pkgPath string) bool { + var goModulesFileFound bool + for _, fileName := range goModulesFilesToCheck { + pathToCheck := filepath.Join(pkgPath, fileName) + if _, err := os.Stat(pathToCheck); err == nil { + goModulesFileFound = true + } + } + + if goModulesFileFound { + log.Println("Project '" + pkgPath + "' is using Go modules for dependencies management") + } + + return goModulesFileFound +} diff --git a/parsers/golang/parser.go b/parsers/golang/parser.go new file mode 100644 index 0000000..ff3c4c8 --- /dev/null +++ b/parsers/golang/parser.go @@ -0,0 +1,67 @@ +package golang + +import ( + // stdlib + "sync" + + // local + "go.dev.pztrn.name/glp/structs" +) + +const ( + // Package managers names. Used in Detect() for flavor returning. + packageManagerGoMod = "go mod" + packageManagerDep = "dep" +) + +// This structure responsible for parsing projects that written in Go. +type golangParser struct{} + +// Detect detects if passed project path can be parsed with this parser +// and additionally detect package manager used. +func (gp *golangParser) Detect(pkgPath string) (bool, string) { + // Go projects usually using go modules or dep for dependencies + // management. + isModules := gp.detectModulesUsage(pkgPath) + if isModules { + return true, packageManagerGoMod + } + + isDep := gp.detectDepUsage(pkgPath) + if isDep { + return true, packageManagerDep + } + + return false, "" +} + +// GetDependencies extracts dependencies from project. +func (gp *golangParser) GetDependencies(flavor string, pkgPath string) []*structs.Dependency { + var deps []*structs.Dependency + + switch flavor { + case packageManagerDep: + deps = gp.getDependenciesFromDep(pkgPath) + } + + // Return early if no dependencies was found. + if len(deps) == 0 { + return nil + } + + // For every dependency we should get additional data - go-import + // and go-source. Asynchronously. + var wg sync.WaitGroup + + for _, dep := range deps { + wg.Add(1) + go func(dep *structs.Dependency) { + getGoData(dep) + wg.Done() + }(dep) + } + + wg.Wait() + + return deps +} diff --git a/parsers/parserinterface/parserinterface.go b/parsers/parserinterface/parserinterface.go new file mode 100644 index 0000000..e8d078c --- /dev/null +++ b/parsers/parserinterface/parserinterface.go @@ -0,0 +1,16 @@ +package parserinterface + +import ( + // local + "go.dev.pztrn.name/glp/structs" +) + +// Interface is a generic parser interface. +type Interface interface { + // Detect should return true if project should be parsed using + // this parser and false otherwise. May optionally return package + // flavor (e.g. dependency management utility name). + Detect(pkgPath string) (bool, string) + // GetDependencies parses project for dependencies. + GetDependencies(flavor string, pkgPath string) []*structs.Dependency +} diff --git a/projecter/exported.go b/projecter/exported.go new file mode 100644 index 0000000..0160690 --- /dev/null +++ b/projecter/exported.go @@ -0,0 +1,88 @@ +package projecter + +import ( + // stdlib + "log" + "strings" + "sync" + + // local + "go.dev.pztrn.name/glp/configuration" + "go.dev.pztrn.name/glp/outputters" + "go.dev.pztrn.name/glp/structs" +) + +var ( + packages []string + + outputFormat string + outputFile string + + projects map[string]*Project + projectsMutex sync.RWMutex +) + +// Initialize initializes package. +func Initialize(pkgs string, outFormat string, outFile string) { + log.Println("Initializing projects handler...") + + packages = strings.Split(pkgs, ",") + projects = make(map[string]*Project) + + outputFormat = outFormat + outputFile = outFile + + log.Println("Packages list that was passed:", packages) +} + +// GetProject returns project by it's path. +func GetProject(path string) *Project { + projectsMutex.RLock() + defer projectsMutex.RUnlock() + + prj, found := projects[path] + if !found { + return nil + } + + return prj +} + +// Parse starts projects parsing. +func Parse() { + // Create project for every passed package. + // This is done in main goroutine and therefore no mutex is used. + for _, pkgPath := range packages { + prj := NewProject(pkgPath) + projects[pkgPath] = prj + } + + if configuration.Cfg.Log.Debug { + log.Printf("Projects generated: %+v\n", projects) + } + + // We should start asynchronous projects parsing. + var wg sync.WaitGroup + + for _, prj := range projects { + wg.Add(1) + go func(prj *Project) { + prj.process() + wg.Done() + }(prj) + } + + // Wait until all projects will be parsed. + wg.Wait() + + // Collect dependencies list from all parsed projects. + var deps []*structs.Dependency + + for _, prj := range projects { + deps = append(deps, prj.GetDeps()...) + } + + outputters.Write(outputFormat, outputFile, deps) + + log.Println("Parsing done") +} diff --git a/projecter/project.go b/projecter/project.go new file mode 100644 index 0000000..1cd31db --- /dev/null +++ b/projecter/project.go @@ -0,0 +1,80 @@ +package projecter + +import ( + // stdlib + "log" + "os" + "path/filepath" + "strings" + + // local + "go.dev.pztrn.name/glp/parsers" + "go.dev.pztrn.name/glp/structs" +) + +// Project represents single project (or package) that was passed via +// -pkgs parameter. +type Project struct { + packagePath string + parserName string + flavor string + + deps []*structs.Dependency +} + +// NewProject creates new project and returns it. +func NewProject(packagePath string) *Project { + p := &Project{} + p.initialize(packagePath) + + return p +} + +// GetDeps returns list of dependencies for project. +func (p *Project) GetDeps() []*structs.Dependency { + return p.deps +} + +// Initializes project. +func (p *Project) initialize(packagePath string) { + p.packagePath = packagePath + + // Prepare package path to be used. + // First - replace "~" with actual home directory. + if strings.Contains(p.packagePath, "~") { + homeDir, err := os.UserHomeDir() + if err != nil { + log.Fatalln("Failed to get user's home directory:", err.Error()) + } + + p.packagePath = strings.Replace(p.packagePath, "~", homeDir, -1) + } + + // Get absolute path. + var err error + p.packagePath, err = filepath.Abs(p.packagePath) + if err != nil { + log.Fatalln("Failed to get absolute path for package '"+p.packagePath+":", err.Error()) + } +} + +// Starts project parsing. +func (p *Project) process() { + // We should determine project type. + p.parserName, p.flavor = parsers.Detect(p.packagePath) + + if p.parserName == "unknown" { + log.Println("Project", p.packagePath, "cannot be parsed with glp") + return + } + + // Lets try to get dependencies, their versions and URLs. + deps, err := parsers.GetDependencies(p.parserName, p.flavor, p.packagePath) + if err != nil { + log.Fatalln("Failed to get dependencies:", err.Error()) + } + + p.deps = deps + + // ToDo: get licensing information. +} diff --git a/structs/dependency.go b/structs/dependency.go new file mode 100644 index 0000000..0cfef03 --- /dev/null +++ b/structs/dependency.go @@ -0,0 +1,18 @@ +package structs + +// Dependency represents single dependency data. +type Dependency struct { + // License is a license name for dependency. + License License + // Name is a dependency name as it appears in package manager's + // lock file or in sources if no package manager is used. + Name string + // Parent is a path to parent package. + Parent string + // VCS is a VCS data obtained for dependency. + VCS VCSData + // Version is a dependency version used in project. + Version string + // URL is a web URL for that dependency (Github, Gitlab, etc.). + URL string +} diff --git a/structs/license.go b/structs/license.go new file mode 100644 index 0000000..1f772d0 --- /dev/null +++ b/structs/license.go @@ -0,0 +1,7 @@ +package structs + +type License struct { + Copyrights string + Name string + URL string +} diff --git a/structs/vcsdata.go b/structs/vcsdata.go new file mode 100644 index 0000000..761e3c2 --- /dev/null +++ b/structs/vcsdata.go @@ -0,0 +1,19 @@ +package structs + +// VCSData describes structure of go-import and go-source data. +type VCSData struct { + // Branch is a VCS branch used. + Branch string + // Revision is a VCS revision used. + Revision string + // SourceURLDirTemplate is a template for sources dirs URLs. E.g.: + // https://sources.dev.pztrn.name/pztrn/glp/src/branch/master{/dir} + SourceURLDirTemplate string + // SourceURLFileTemplate is a template for sources files URLs. E.g.: + // https://sources.dev.pztrn.name/pztrn/glp/src/branch/master{/dir}/{file}#L{line} + SourceURLFileTemplate string + // VCS is a VCS name (e.g. "git"). + VCS string + // VCSPath is a VCS repository path. + VCSPath string +}