diff --git a/check.go b/check.go index 64cc9fc..a2c6fc2 100644 --- a/check.go +++ b/check.go @@ -56,7 +56,7 @@ func (a *goBlog) checkLinks(w io.Writer, posts ...*post) error { // Create HTTP client client := &http.Client{ Timeout: 30 * time.Second, - Transport: httpcachetransport.NewHttpCacheTransport(gzhttp.Transport(&http.Transport{ + Transport: httpcachetransport.NewHttpCacheTransportNoBody(gzhttp.Transport(&http.Transport{ DisableKeepAlives: true, MaxConnsPerHost: 1, }), cache, 60*time.Minute), } @@ -66,7 +66,7 @@ func (a *goBlog) checkLinks(w io.Writer, posts ...*post) error { status int err error } - p := pool.NewWithResults[*checkresult]().WithMaxGoroutines(5).WithContext(cancelContext) + p := pool.NewWithResults[*checkresult]().WithMaxGoroutines(10).WithContext(cancelContext) for _, link := range allLinks { link := link p.Go(func(ctx context.Context) (result *checkresult, _ error) { diff --git a/pkgs/httpcachetransport/httpCacheTransport.go b/pkgs/httpcachetransport/httpCacheTransport.go index 656f1b3..42d5aae 100644 --- a/pkgs/httpcachetransport/httpCacheTransport.go +++ b/pkgs/httpcachetransport/httpCacheTransport.go @@ -14,6 +14,7 @@ type httpCacheTransport struct { parent http.RoundTripper ristrettoCache *ristretto.Cache ttl time.Duration + body bool } func (t *httpCacheTransport) RoundTrip(r *http.Request) (*http.Response, error) { @@ -27,7 +28,7 @@ func (t *httpCacheTransport) RoundTrip(r *http.Request) (*http.Response, error) } resp, err := t.parent.RoundTrip(r) if err == nil && t.ristrettoCache != nil { - respBytes, err := httputil.DumpResponse(resp, true) + respBytes, err := httputil.DumpResponse(resp, t.body) if err != nil { return resp, err } @@ -41,5 +42,10 @@ func (t *httpCacheTransport) RoundTrip(r *http.Request) (*http.Response, error) // Creates a new http.RoundTripper that caches all // request responses (by the request URL) in ristretto. func NewHttpCacheTransport(parent http.RoundTripper, ristrettoCache *ristretto.Cache, ttl time.Duration) http.RoundTripper { - return &httpCacheTransport{parent, ristrettoCache, ttl} + return &httpCacheTransport{parent, ristrettoCache, ttl, true} +} + +// Like NewHttpCacheTransport but doesn't cache body +func NewHttpCacheTransportNoBody(parent http.RoundTripper, ristrettoCache *ristretto.Cache, ttl time.Duration) http.RoundTripper { + return &httpCacheTransport{parent, ristrettoCache, ttl, false} }