glp/httpclient/exported.go

package httpclient

import (
	// stdlib
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"sync"
	"time"

	// local
	"go.dev.pztrn.name/glp/configuration"
)

const (
	defaultTimeoutInSeconds = 20
	perDomainRequestsLimit  = 5
)

var (
	httpClient = &http.Client{
		Timeout: time.Second * defaultTimeoutInSeconds,
		Transport: &http.Transport{
			DialContext: (&net.Dialer{
				Timeout:   time.Second * defaultTimeoutInSeconds,
				DualStack: true,
			}).DialContext,
			ExpectContinueTimeout: time.Second * 5,
			Proxy:                 http.ProxyFromEnvironment,
			ResponseHeaderTimeout: time.Second * defaultTimeoutInSeconds,
			TLSHandshakeTimeout:   time.Second * 5,
		},
	}

	perDomainRequests      map[string]int
	perDomainRequestsMutex sync.Mutex
)

// Initialize initializes package.
func Initialize() {
	log.Println("Initializing HTTP client...")

	perDomainRequests = make(map[string]int)
}

// GET executes GET request and returns body.
func GET(request *http.Request) []byte {
	for {
		perDomainRequestsMutex.Lock()
		currentlyRunning, found := perDomainRequests[request.URL.Host]
		perDomainRequestsMutex.Unlock()

		if !found {
			break
		}

		if currentlyRunning >= perDomainRequestsLimit {
			time.Sleep(time.Second * 1)
			continue
		}

		break
	}

	perDomainRequestsMutex.Lock()

	_, found := perDomainRequests[request.URL.Host]
	if !found {
		perDomainRequests[request.URL.Host] = 1
	} else {
		perDomainRequests[request.URL.Host]++
	}

	perDomainRequestsMutex.Unlock()

	defer func() {
		perDomainRequestsMutex.Lock()

		perDomainRequests[request.URL.Host]--

		perDomainRequestsMutex.Unlock()
	}()

	if configuration.Cfg.Log.Debug {
		log.Println("Executing request:", request.URL.String())
	}

	var (
		requestsCount = 0
		response      *http.Response
	)

	for {
		if requestsCount == 3 {
			log.Printf("Failed to execute request %s: tried 3 times and got errors. Skipping.", request.URL.String())
			return nil
		}

		var err error

		response, err = httpClient.Do(request)
		if err != nil {
			log.Printf("Failed to execute request %s: %s\n", request.URL.String(), err.Error())
			requestsCount++
			time.Sleep(time.Second * 1)
			continue
		}

		break
	}

	respBody, err1 := ioutil.ReadAll(response.Body)
	response.Body.Close()

	if err1 != nil {
		log.Printf("Failed to read response body %s: %s\n", request.URL.String(), err1.Error())
		return nil
	}

	return respBody
}