Rework media compression: simplify and use memory buffer instead of temporary file

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/alecthomas/chroma v0.9.4
 	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
 	github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
+	github.com/carlmjohnson/requests v0.21.13
 	github.com/cretz/bine v0.2.0
 	github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
 	github.com/dgraph-io/ristretto v0.1.0

go.sum

@@ -63,6 +63,8 @@ github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8
 github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 h1:t8KYCwSKsOEZBFELI4Pn/phbp38iJ1RRAkDFNin1aak=
 github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
+github.com/carlmjohnson/requests v0.21.13 h1:p9DiBwbrLG8uA67YPOrfGMG1ZRzRyPBaO9hXQpX+Ork=
+github.com/carlmjohnson/requests v0.21.13/go.mod h1:Hw4fFOk3xDlHQbNRTGo4oc52TUTpVEq93sNy/H+mrQM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=

mediaCompression.go

@@ -2,14 +2,13 @@ package main
 
 import (
 	"bytes"
-	"encoding/json"
+	"context"
 	"errors"
 	"fmt"
-	"io"
+	"log"
 	"net/http"
-	"os"
 
-	"go.goblog.app/app/pkgs/contenttype"
+	"github.com/carlmjohnson/requests"
 )
 
 const defaultCompressionWidth = 2000
@@ -34,10 +33,10 @@ func (a *goBlog) compressMediaFile(url string) (location string, err error) {
 }
 
 func (a *goBlog) initMediaCompressors() {
-	config := a.cfg.Micropub.MediaStorage
+	if a.cfg.Micropub == nil || a.cfg.Micropub.MediaStorage == nil {
-	if config == nil {
 		return
 	}
+	config := a.cfg.Micropub.MediaStorage
 	if key := config.ShortPixelKey; key != "" {
 		a.compressors = append(a.compressors, &shortpixel{key})
 	}
@@ -53,178 +52,130 @@ type shortpixel struct {
 	key string
 }
 
-var _ mediaCompression = &shortpixel{}
+func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (string, error) {
-
-func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (location string, err error) {
 	// Check url
 	fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png")
 	if !allowed {
 		return "", nil
 	}
 	// Compress
-	j, _ := json.Marshal(map[string]interface{}{
+	var imgBuffer bytes.Buffer
-		"key":            sp.key,
+	err := requests.
-		"plugin_version": "GB001",
+		URL("https://api.shortpixel.com/v2/reducer-sync.php").
-		"lossy":          1,
+		Client(hc).
-		"resize":         3,
+		Post().
-		"resize_width":   defaultCompressionWidth,
+		BodyJSON(map[string]interface{}{
-		"resize_height":  defaultCompressionHeight,
+			"key":            sp.key,
-		"cmyk2rgb":       1,
+			"plugin_version": "GB001",
-		"keep_exif":      0,
+			"lossy":          1,
-		"url":            url,
+			"resize":         3,
-	})
+			"resize_width":   defaultCompressionWidth,
-	req, err := http.NewRequest(http.MethodPut, "https://api.shortpixel.com/v2/reducer-sync.php", bytes.NewReader(j))
+			"resize_height":  defaultCompressionHeight,
+			"cmyk2rgb":       1,
+			"keep_exif":      0,
+			"url":            url,
+		}).
+		ToBytesBuffer(&imgBuffer).
+		Fetch(context.Background())
 	if err != nil {
-		return "", err
+		log.Println("Shortpixel error:", err.Error())
-	}
+		return "", errors.New("failed to compress image using shortpixel")
-	resp, err := hc.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("shortpixel failed to compress image, status code %d", resp.StatusCode)
-	}
-	tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension)
-	if err != nil {
-		return "", err
-	}
-	defer func() {
-		_ = tmpFile.Close()
-		_ = os.Remove(tmpFile.Name())
-	}()
-	if _, err = io.Copy(tmpFile, resp.Body); err != nil {
-		return "", err
-	}
-	fileName, err := getSHA256(tmpFile)
-	if err != nil {
-		return "", err
 	}
 	// Upload compressed file
-	location, err = upload(fileName+"."+fileExtension, tmpFile)
+	return uploadCompressedFile(fileExtension, &imgBuffer, upload)
-	return
 }
 
 type tinify struct {
 	key string
 }
 
-var _ mediaCompression = &tinify{}
+func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (string, error) {
-
-func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (location string, err error) {
 	// Check url
 	fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png")
 	if !allowed {
 		return "", nil
 	}
 	// Compress
-	j, _ := json.Marshal(map[string]interface{}{
+	compressedLocation := ""
-		"source": map[string]interface{}{
+	err := requests.
-			"url": url,
+		URL("https://api.tinify.com/shrink").
-		},
+		Client(hc).
-	})
+		Post().
-	req, err := http.NewRequest(http.MethodPost, "https://api.tinify.com/shrink", bytes.NewReader(j))
+		BasicAuth("api", tf.key).
+		BodyJSON(map[string]interface{}{
+			"source": map[string]interface{}{
+				"url": url,
+			},
+		}).
+		Handle(func(r *http.Response) error {
+			compressedLocation = r.Header.Get("Location")
+			if compressedLocation == "" {
+				return errors.New("location header missing")
+			}
+			return nil
+		}).
+		Fetch(context.Background())
 	if err != nil {
-		return "", err
+		log.Println("Tinify error:", err.Error())
-	}
+		return "", errors.New("failed to compress image using tinify")
-	req.SetBasicAuth("api", tf.key)
-	req.Header.Set(contentType, contenttype.JSON)
-	resp, err := hc.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusCreated {
-		return "", fmt.Errorf("failed to compress image, status code %d", resp.StatusCode)
-	}
-	compressedLocation := resp.Header.Get("Location")
-	if compressedLocation == "" {
-		return "", errors.New("tinify didn't return compressed location")
 	}
 	// Resize and download image
-	j, _ = json.Marshal(map[string]interface{}{
+	var imgBuffer bytes.Buffer
-		"resize": map[string]interface{}{
+	err = requests.
-			"method": "fit",
+		URL(compressedLocation).
-			"width":  defaultCompressionWidth,
+		Client(hc).
-			"height": defaultCompressionHeight,
+		Post().
-		},
+		BasicAuth("api", tf.key).
-	})
+		BodyJSON(map[string]interface{}{
-	downloadReq, err := http.NewRequest(http.MethodPost, compressedLocation, bytes.NewReader(j))
+			"resize": map[string]interface{}{
+				"method": "fit",
+				"width":  defaultCompressionWidth,
+				"height": defaultCompressionHeight,
+			},
+		}).
+		ToBytesBuffer(&imgBuffer).
+		Fetch(context.Background())
 	if err != nil {
-		return "", err
+		log.Println("Tinify error:", err.Error())
-	}
+		return "", errors.New("failed to compress image using tinify")
-	downloadReq.SetBasicAuth("api", tf.key)
-	downloadReq.Header.Set(contentType, contenttype.JSON)
-	downloadResp, err := hc.Do(downloadReq)
-	if err != nil {
-		return "", err
-	}
-	defer downloadResp.Body.Close()
-	if downloadResp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("tinify failed to resize image, status code %d", downloadResp.StatusCode)
-	}
-	tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension)
-	if err != nil {
-		return "", err
-	}
-	defer func() {
-		_ = tmpFile.Close()
-		_ = os.Remove(tmpFile.Name())
-	}()
-	if _, err = io.Copy(tmpFile, downloadResp.Body); err != nil {
-		return "", err
-	}
-	fileName, err := getSHA256(tmpFile)
-	if err != nil {
-		return "", err
 	}
 	// Upload compressed file
-	location, err = upload(fileName+"."+fileExtension, tmpFile)
+	return uploadCompressedFile(fileExtension, &imgBuffer, upload)
-	return
 }
 
-type cloudflare struct {
+type cloudflare struct{}
-}
 
-var _ mediaCompression = &cloudflare{}
+func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (string, error) {
-
-func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (location string, err error) {
 	// Check url
-	_, allowed := urlHasExt(url, "jpg", "jpeg", "png")
+	if _, allowed := urlHasExt(url, "jpg", "jpeg", "png"); !allowed {
-	if !allowed {
 		return "", nil
 	}
 	// Force jpeg
 	fileExtension := "jpeg"
 	// Compress
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=%d,h=%d/%s", defaultCompressionWidth, defaultCompressionHeight, url), nil)
+	var imgBuffer bytes.Buffer
+	err := requests.
+		URL(fmt.Sprintf("https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=%d,h=%d/%s", defaultCompressionWidth, defaultCompressionHeight, url)).
+		Client(hc).
+		Get().
+		ToBytesBuffer(&imgBuffer).
+		Fetch(context.Background())
 	if err != nil {
-		return "", err
+		log.Println("Cloudflare error:", err.Error())
+		return "", errors.New("failed to compress image using cloudflare")
 	}
-	resp, err := hc.Do(req)
+	// Upload compressed file
-	if err != nil {
+	return uploadCompressedFile(fileExtension, &imgBuffer, upload)
-		return "", err
+}
-	}
+
-	defer resp.Body.Close()
+func uploadCompressedFile(fileExtension string, imgBuffer *bytes.Buffer, upload mediaStorageSaveFunc) (string, error) {
-	if resp.StatusCode != http.StatusOK {
+	// Create reader from buffer
-		return "", fmt.Errorf("cloudflare failed to compress image, status code %d", resp.StatusCode)
+	imgReader := bytes.NewReader(imgBuffer.Bytes())
-	}
+	// Get hash of compressed file
-	tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension)
+	fileName, err := getSHA256(imgReader)
-	if err != nil {
-		return "", err
-	}
-	defer func() {
-		_ = tmpFile.Close()
-		_ = os.Remove(tmpFile.Name())
-	}()
-	if _, err = io.Copy(tmpFile, resp.Body); err != nil {
-		return "", err
-	}
-	fileName, err := getSHA256(tmpFile)
 	if err != nil {
 		return "", err
 	}
 	// Upload compressed file
-	location, err = upload(fileName+"."+fileExtension, tmpFile)
+	return upload(fileName+"."+fileExtension, imgReader)
-	return
 }