From 1be1564eb73399ed84422357cc1f16451576ceba Mon Sep 17 00:00:00 2001 From: Jan-Lukas Else Date: Fri, 24 Dec 2021 12:58:14 +0100 Subject: [PATCH] Rework media compression: simplify and use memory buffer instead of temporary file --- go.mod | 1 + go.sum | 2 + mediaCompression.go | 227 +++++++++++++++++--------------------------- 3 files changed, 92 insertions(+), 138 deletions(-) diff --git a/go.mod b/go.mod index 110020f..1a7154f 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/alecthomas/chroma v0.9.4 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 + github.com/carlmjohnson/requests v0.21.13 github.com/cretz/bine v0.2.0 github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f github.com/dgraph-io/ristretto v0.1.0 diff --git a/go.sum b/go.sum index 7c93879..9351bcb 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,8 @@ github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8 github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 h1:t8KYCwSKsOEZBFELI4Pn/phbp38iJ1RRAkDFNin1aak= github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= +github.com/carlmjohnson/requests v0.21.13 h1:p9DiBwbrLG8uA67YPOrfGMG1ZRzRyPBaO9hXQpX+Ork= +github.com/carlmjohnson/requests v0.21.13/go.mod h1:Hw4fFOk3xDlHQbNRTGo4oc52TUTpVEq93sNy/H+mrQM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= diff --git a/mediaCompression.go b/mediaCompression.go index ddba66f..aeb93bb 100644 --- a/mediaCompression.go +++ b/mediaCompression.go @@ -2,14 +2,13 @@ package main import ( "bytes" - "encoding/json" + "context" "errors" "fmt" - "io" + "log" "net/http" - "os" - "go.goblog.app/app/pkgs/contenttype" + "github.com/carlmjohnson/requests" ) const defaultCompressionWidth = 2000 @@ -34,10 +33,10 @@ func (a *goBlog) compressMediaFile(url string) (location string, err error) { } func (a *goBlog) initMediaCompressors() { - config := a.cfg.Micropub.MediaStorage - if config == nil { + if a.cfg.Micropub == nil || a.cfg.Micropub.MediaStorage == nil { return } + config := a.cfg.Micropub.MediaStorage if key := config.ShortPixelKey; key != "" { a.compressors = append(a.compressors, &shortpixel{key}) } @@ -53,178 +52,130 @@ type shortpixel struct { key string } -var _ mediaCompression = &shortpixel{} - -func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (location string, err error) { +func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (string, error) { // Check url fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png") if !allowed { return "", nil } // Compress - j, _ := json.Marshal(map[string]interface{}{ - "key": sp.key, - "plugin_version": "GB001", - "lossy": 1, - "resize": 3, - "resize_width": defaultCompressionWidth, - "resize_height": defaultCompressionHeight, - "cmyk2rgb": 1, - "keep_exif": 0, - "url": url, - }) - req, err := http.NewRequest(http.MethodPut, "https://api.shortpixel.com/v2/reducer-sync.php", bytes.NewReader(j)) + var imgBuffer bytes.Buffer + err := requests. + URL("https://api.shortpixel.com/v2/reducer-sync.php"). + Client(hc). + Post(). + BodyJSON(map[string]interface{}{ + "key": sp.key, + "plugin_version": "GB001", + "lossy": 1, + "resize": 3, + "resize_width": defaultCompressionWidth, + "resize_height": defaultCompressionHeight, + "cmyk2rgb": 1, + "keep_exif": 0, + "url": url, + }). + ToBytesBuffer(&imgBuffer). + Fetch(context.Background()) if err != nil { - return "", err - } - resp, err := hc.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("shortpixel failed to compress image, status code %d", resp.StatusCode) - } - tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension) - if err != nil { - return "", err - } - defer func() { - _ = tmpFile.Close() - _ = os.Remove(tmpFile.Name()) - }() - if _, err = io.Copy(tmpFile, resp.Body); err != nil { - return "", err - } - fileName, err := getSHA256(tmpFile) - if err != nil { - return "", err + log.Println("Shortpixel error:", err.Error()) + return "", errors.New("failed to compress image using shortpixel") } // Upload compressed file - location, err = upload(fileName+"."+fileExtension, tmpFile) - return + return uploadCompressedFile(fileExtension, &imgBuffer, upload) } type tinify struct { key string } -var _ mediaCompression = &tinify{} - -func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (location string, err error) { +func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (string, error) { // Check url fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png") if !allowed { return "", nil } // Compress - j, _ := json.Marshal(map[string]interface{}{ - "source": map[string]interface{}{ - "url": url, - }, - }) - req, err := http.NewRequest(http.MethodPost, "https://api.tinify.com/shrink", bytes.NewReader(j)) + compressedLocation := "" + err := requests. + URL("https://api.tinify.com/shrink"). + Client(hc). + Post(). + BasicAuth("api", tf.key). + BodyJSON(map[string]interface{}{ + "source": map[string]interface{}{ + "url": url, + }, + }). + Handle(func(r *http.Response) error { + compressedLocation = r.Header.Get("Location") + if compressedLocation == "" { + return errors.New("location header missing") + } + return nil + }). + Fetch(context.Background()) if err != nil { - return "", err - } - req.SetBasicAuth("api", tf.key) - req.Header.Set(contentType, contenttype.JSON) - resp, err := hc.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusCreated { - return "", fmt.Errorf("failed to compress image, status code %d", resp.StatusCode) - } - compressedLocation := resp.Header.Get("Location") - if compressedLocation == "" { - return "", errors.New("tinify didn't return compressed location") + log.Println("Tinify error:", err.Error()) + return "", errors.New("failed to compress image using tinify") } // Resize and download image - j, _ = json.Marshal(map[string]interface{}{ - "resize": map[string]interface{}{ - "method": "fit", - "width": defaultCompressionWidth, - "height": defaultCompressionHeight, - }, - }) - downloadReq, err := http.NewRequest(http.MethodPost, compressedLocation, bytes.NewReader(j)) + var imgBuffer bytes.Buffer + err = requests. + URL(compressedLocation). + Client(hc). + Post(). + BasicAuth("api", tf.key). + BodyJSON(map[string]interface{}{ + "resize": map[string]interface{}{ + "method": "fit", + "width": defaultCompressionWidth, + "height": defaultCompressionHeight, + }, + }). + ToBytesBuffer(&imgBuffer). + Fetch(context.Background()) if err != nil { - return "", err - } - downloadReq.SetBasicAuth("api", tf.key) - downloadReq.Header.Set(contentType, contenttype.JSON) - downloadResp, err := hc.Do(downloadReq) - if err != nil { - return "", err - } - defer downloadResp.Body.Close() - if downloadResp.StatusCode != http.StatusOK { - return "", fmt.Errorf("tinify failed to resize image, status code %d", downloadResp.StatusCode) - } - tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension) - if err != nil { - return "", err - } - defer func() { - _ = tmpFile.Close() - _ = os.Remove(tmpFile.Name()) - }() - if _, err = io.Copy(tmpFile, downloadResp.Body); err != nil { - return "", err - } - fileName, err := getSHA256(tmpFile) - if err != nil { - return "", err + log.Println("Tinify error:", err.Error()) + return "", errors.New("failed to compress image using tinify") } // Upload compressed file - location, err = upload(fileName+"."+fileExtension, tmpFile) - return + return uploadCompressedFile(fileExtension, &imgBuffer, upload) } -type cloudflare struct { -} +type cloudflare struct{} -var _ mediaCompression = &cloudflare{} - -func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (location string, err error) { +func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (string, error) { // Check url - _, allowed := urlHasExt(url, "jpg", "jpeg", "png") - if !allowed { + if _, allowed := urlHasExt(url, "jpg", "jpeg", "png"); !allowed { return "", nil } // Force jpeg fileExtension := "jpeg" // Compress - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=%d,h=%d/%s", defaultCompressionWidth, defaultCompressionHeight, url), nil) + var imgBuffer bytes.Buffer + err := requests. + URL(fmt.Sprintf("https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=%d,h=%d/%s", defaultCompressionWidth, defaultCompressionHeight, url)). + Client(hc). + Get(). + ToBytesBuffer(&imgBuffer). + Fetch(context.Background()) if err != nil { - return "", err + log.Println("Cloudflare error:", err.Error()) + return "", errors.New("failed to compress image using cloudflare") } - resp, err := hc.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("cloudflare failed to compress image, status code %d", resp.StatusCode) - } - tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension) - if err != nil { - return "", err - } - defer func() { - _ = tmpFile.Close() - _ = os.Remove(tmpFile.Name()) - }() - if _, err = io.Copy(tmpFile, resp.Body); err != nil { - return "", err - } - fileName, err := getSHA256(tmpFile) + // Upload compressed file + return uploadCompressedFile(fileExtension, &imgBuffer, upload) +} + +func uploadCompressedFile(fileExtension string, imgBuffer *bytes.Buffer, upload mediaStorageSaveFunc) (string, error) { + // Create reader from buffer + imgReader := bytes.NewReader(imgBuffer.Bytes()) + // Get hash of compressed file + fileName, err := getSHA256(imgReader) if err != nil { return "", err } // Upload compressed file - location, err = upload(fileName+"."+fileExtension, tmpFile) - return + return upload(fileName+"."+fileExtension, imgReader) }