Initial commit.

2020-02-06 15:09:25 +05:00
commit a7bf1f29ba
22 changed files with 1049 additions and 0 deletions
--- a/httpclient/exported.go
+++ b/httpclient/exported.go
@@ -0,0 +1,122 @@
+package httpclient
+
+import (
+	// stdlib
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	// local
+	"go.dev.pztrn.name/glp/configuration"
+)
+
+const (
+	defaultTimeoutInSeconds = 20
+	perDomainRequestsLimit  = 5
+)
+
+var (
+	httpClient = &http.Client{
+		Timeout: time.Second * defaultTimeoutInSeconds,
+		Transport: &http.Transport{
+			DialContext: (&net.Dialer{
+				Timeout:   time.Second * defaultTimeoutInSeconds,
+				DualStack: true,
+			}).DialContext,
+			ExpectContinueTimeout: time.Second * 5,
+			Proxy:                 http.ProxyFromEnvironment,
+			ResponseHeaderTimeout: time.Second * defaultTimeoutInSeconds,
+			TLSHandshakeTimeout:   time.Second * 5,
+		},
+	}
+
+	perDomainRequests      map[string]int
+	perDomainRequestsMutex sync.Mutex
+)
+
+// Initialize initializes package.
+func Initialize() {
+	log.Println("Initializing HTTP client...")
+
+	perDomainRequests = make(map[string]int)
+}
+
+// GET executes GET request and returns body.
+func GET(request *http.Request) []byte {
+	for {
+		perDomainRequestsMutex.Lock()
+		currentlyRunning, found := perDomainRequests[request.URL.Host]
+		perDomainRequestsMutex.Unlock()
+
+		if !found {
+			break
+		}
+
+		if currentlyRunning >= perDomainRequestsLimit {
+			time.Sleep(time.Second * 1)
+			continue
+		}
+
+		break
+	}
+
+	perDomainRequestsMutex.Lock()
+
+	_, found := perDomainRequests[request.URL.Host]
+	if !found {
+		perDomainRequests[request.URL.Host] = 1
+	} else {
+		perDomainRequests[request.URL.Host]++
+	}
+
+	perDomainRequestsMutex.Unlock()
+
+	defer func() {
+		perDomainRequestsMutex.Lock()
+
+		perDomainRequests[request.URL.Host]--
+
+		perDomainRequestsMutex.Unlock()
+	}()
+
+	if configuration.Cfg.Log.Debug {
+		log.Println("Executing request:", request.URL.String())
+	}
+
+	var (
+		requestsCount = 0
+		response      *http.Response
+	)
+
+	for {
+		if requestsCount == 3 {
+			log.Printf("Failed to execute request %s: tried 3 times and got errors. Skipping.", request.URL.String())
+			return nil
+		}
+
+		var err error
+
+		response, err = httpClient.Do(request)
+		if err != nil {
+			log.Printf("Failed to execute request %s: %s\n", request.URL.String(), err.Error())
+			requestsCount++
+			time.Sleep(time.Second * 1)
+			continue
+		}
+
+		break
+	}
+
+	respBody, err1 := ioutil.ReadAll(response.Body)
+	response.Body.Close()
+
+	if err1 != nil {
+		log.Printf("Failed to read response body %s: %s\n", request.URL.String(), err1.Error())
+		return nil
+	}
+
+	return respBody
+}