glp/httpclient/exported.go

123 lines
2.3 KiB
Go
Raw Normal View History

2020-02-06 15:09:25 +05:00
package httpclient
import (
// stdlib
"io/ioutil"
"log"
"net"
"net/http"
"sync"
"time"
// local
"go.dev.pztrn.name/glp/configuration"
)
const (
defaultTimeoutInSeconds = 20
perDomainRequestsLimit = 5
)
var (
httpClient = &http.Client{
Timeout: time.Second * defaultTimeoutInSeconds,
Transport: &http.Transport{
DialContext: (&net.Dialer{
Timeout: time.Second * defaultTimeoutInSeconds,
DualStack: true,
}).DialContext,
ExpectContinueTimeout: time.Second * 5,
Proxy: http.ProxyFromEnvironment,
ResponseHeaderTimeout: time.Second * defaultTimeoutInSeconds,
TLSHandshakeTimeout: time.Second * 5,
},
}
perDomainRequests map[string]int
perDomainRequestsMutex sync.Mutex
)
// Initialize initializes package.
func Initialize() {
log.Println("Initializing HTTP client...")
perDomainRequests = make(map[string]int)
}
// GET executes GET request and returns body.
func GET(request *http.Request) []byte {
for {
perDomainRequestsMutex.Lock()
currentlyRunning, found := perDomainRequests[request.URL.Host]
perDomainRequestsMutex.Unlock()
if !found {
break
}
if currentlyRunning >= perDomainRequestsLimit {
time.Sleep(time.Second * 1)
continue
}
break
}
perDomainRequestsMutex.Lock()
_, found := perDomainRequests[request.URL.Host]
if !found {
perDomainRequests[request.URL.Host] = 1
} else {
perDomainRequests[request.URL.Host]++
}
perDomainRequestsMutex.Unlock()
defer func() {
perDomainRequestsMutex.Lock()
perDomainRequests[request.URL.Host]--
perDomainRequestsMutex.Unlock()
}()
if configuration.Cfg.Log.Debug {
log.Println("Executing request:", request.URL.String())
}
var (
requestsCount = 0
response *http.Response
)
for {
if requestsCount == 3 {
log.Printf("Failed to execute request %s: tried 3 times and got errors. Skipping.", request.URL.String())
return nil
}
var err error
response, err = httpClient.Do(request)
if err != nil {
log.Printf("Failed to execute request %s: %s\n", request.URL.String(), err.Error())
requestsCount++
time.Sleep(time.Second * 1)
continue
}
break
}
respBody, err1 := ioutil.ReadAll(response.Body)
response.Body.Close()
if err1 != nil {
log.Printf("Failed to read response body %s: %s\n", request.URL.String(), err1.Error())
return nil
}
return respBody
}