Don't cache response body for check

This commit is contained in:
Jan-Lukas Else 2023-01-29 22:15:55 +01:00
parent ac94e7d124
commit 4ab7be14b6
2 changed files with 10 additions and 4 deletions

View File

@ -56,7 +56,7 @@ func (a *goBlog) checkLinks(w io.Writer, posts ...*post) error {
// Create HTTP client
client := &http.Client{
Timeout: 30 * time.Second,
Transport: httpcachetransport.NewHttpCacheTransport(gzhttp.Transport(&http.Transport{
Transport: httpcachetransport.NewHttpCacheTransportNoBody(gzhttp.Transport(&http.Transport{
DisableKeepAlives: true, MaxConnsPerHost: 1,
}), cache, 60*time.Minute),
}
@ -66,7 +66,7 @@ func (a *goBlog) checkLinks(w io.Writer, posts ...*post) error {
status int
err error
}
p := pool.NewWithResults[*checkresult]().WithMaxGoroutines(5).WithContext(cancelContext)
p := pool.NewWithResults[*checkresult]().WithMaxGoroutines(10).WithContext(cancelContext)
for _, link := range allLinks {
link := link
p.Go(func(ctx context.Context) (result *checkresult, _ error) {

View File

@ -14,6 +14,7 @@ type httpCacheTransport struct {
parent http.RoundTripper
ristrettoCache *ristretto.Cache
ttl time.Duration
body bool
}
func (t *httpCacheTransport) RoundTrip(r *http.Request) (*http.Response, error) {
@ -27,7 +28,7 @@ func (t *httpCacheTransport) RoundTrip(r *http.Request) (*http.Response, error)
}
resp, err := t.parent.RoundTrip(r)
if err == nil && t.ristrettoCache != nil {
respBytes, err := httputil.DumpResponse(resp, true)
respBytes, err := httputil.DumpResponse(resp, t.body)
if err != nil {
return resp, err
}
@ -41,5 +42,10 @@ func (t *httpCacheTransport) RoundTrip(r *http.Request) (*http.Response, error)
// Creates a new http.RoundTripper that caches all
// request responses (by the request URL) in ristretto.
func NewHttpCacheTransport(parent http.RoundTripper, ristrettoCache *ristretto.Cache, ttl time.Duration) http.RoundTripper {
return &httpCacheTransport{parent, ristrettoCache, ttl}
return &httpCacheTransport{parent, ristrettoCache, ttl, true}
}
// Like NewHttpCacheTransport but doesn't cache body
func NewHttpCacheTransportNoBody(parent http.RoundTripper, ristrettoCache *ristretto.Cache, ttl time.Duration) http.RoundTripper {
return &httpCacheTransport{parent, ristrettoCache, ttl, false}
}