Rework media compression: simplify and use memory buffer instead of temporary file

This commit is contained in:
Jan-Lukas Else 2021-12-24 12:58:14 +01:00
parent a1a88cb6df
commit 1be1564eb7
3 changed files with 92 additions and 138 deletions

1
go.mod
View File

@ -11,6 +11,7 @@ require (
github.com/alecthomas/chroma v0.9.4
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
github.com/carlmjohnson/requests v0.21.13
github.com/cretz/bine v0.2.0
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
github.com/dgraph-io/ristretto v0.1.0

2
go.sum
View File

@ -63,6 +63,8 @@ github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 h1:t8KYCwSKsOEZBFELI4Pn/phbp38iJ1RRAkDFNin1aak=
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
github.com/carlmjohnson/requests v0.21.13 h1:p9DiBwbrLG8uA67YPOrfGMG1ZRzRyPBaO9hXQpX+Ork=
github.com/carlmjohnson/requests v0.21.13/go.mod h1:Hw4fFOk3xDlHQbNRTGo4oc52TUTpVEq93sNy/H+mrQM=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=

View File

@ -2,14 +2,13 @@ package main
import (
"bytes"
"encoding/json"
"context"
"errors"
"fmt"
"io"
"log"
"net/http"
"os"
"go.goblog.app/app/pkgs/contenttype"
"github.com/carlmjohnson/requests"
)
const defaultCompressionWidth = 2000
@ -34,10 +33,10 @@ func (a *goBlog) compressMediaFile(url string) (location string, err error) {
}
func (a *goBlog) initMediaCompressors() {
config := a.cfg.Micropub.MediaStorage
if config == nil {
if a.cfg.Micropub == nil || a.cfg.Micropub.MediaStorage == nil {
return
}
config := a.cfg.Micropub.MediaStorage
if key := config.ShortPixelKey; key != "" {
a.compressors = append(a.compressors, &shortpixel{key})
}
@ -53,178 +52,130 @@ type shortpixel struct {
key string
}
var _ mediaCompression = &shortpixel{}
func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (location string, err error) {
func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (string, error) {
// Check url
fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png")
if !allowed {
return "", nil
}
// Compress
j, _ := json.Marshal(map[string]interface{}{
"key": sp.key,
"plugin_version": "GB001",
"lossy": 1,
"resize": 3,
"resize_width": defaultCompressionWidth,
"resize_height": defaultCompressionHeight,
"cmyk2rgb": 1,
"keep_exif": 0,
"url": url,
})
req, err := http.NewRequest(http.MethodPut, "https://api.shortpixel.com/v2/reducer-sync.php", bytes.NewReader(j))
var imgBuffer bytes.Buffer
err := requests.
URL("https://api.shortpixel.com/v2/reducer-sync.php").
Client(hc).
Post().
BodyJSON(map[string]interface{}{
"key": sp.key,
"plugin_version": "GB001",
"lossy": 1,
"resize": 3,
"resize_width": defaultCompressionWidth,
"resize_height": defaultCompressionHeight,
"cmyk2rgb": 1,
"keep_exif": 0,
"url": url,
}).
ToBytesBuffer(&imgBuffer).
Fetch(context.Background())
if err != nil {
return "", err
}
resp, err := hc.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("shortpixel failed to compress image, status code %d", resp.StatusCode)
}
tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension)
if err != nil {
return "", err
}
defer func() {
_ = tmpFile.Close()
_ = os.Remove(tmpFile.Name())
}()
if _, err = io.Copy(tmpFile, resp.Body); err != nil {
return "", err
}
fileName, err := getSHA256(tmpFile)
if err != nil {
return "", err
log.Println("Shortpixel error:", err.Error())
return "", errors.New("failed to compress image using shortpixel")
}
// Upload compressed file
location, err = upload(fileName+"."+fileExtension, tmpFile)
return
return uploadCompressedFile(fileExtension, &imgBuffer, upload)
}
type tinify struct {
key string
}
var _ mediaCompression = &tinify{}
func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (location string, err error) {
func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (string, error) {
// Check url
fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png")
if !allowed {
return "", nil
}
// Compress
j, _ := json.Marshal(map[string]interface{}{
"source": map[string]interface{}{
"url": url,
},
})
req, err := http.NewRequest(http.MethodPost, "https://api.tinify.com/shrink", bytes.NewReader(j))
compressedLocation := ""
err := requests.
URL("https://api.tinify.com/shrink").
Client(hc).
Post().
BasicAuth("api", tf.key).
BodyJSON(map[string]interface{}{
"source": map[string]interface{}{
"url": url,
},
}).
Handle(func(r *http.Response) error {
compressedLocation = r.Header.Get("Location")
if compressedLocation == "" {
return errors.New("location header missing")
}
return nil
}).
Fetch(context.Background())
if err != nil {
return "", err
}
req.SetBasicAuth("api", tf.key)
req.Header.Set(contentType, contenttype.JSON)
resp, err := hc.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusCreated {
return "", fmt.Errorf("failed to compress image, status code %d", resp.StatusCode)
}
compressedLocation := resp.Header.Get("Location")
if compressedLocation == "" {
return "", errors.New("tinify didn't return compressed location")
log.Println("Tinify error:", err.Error())
return "", errors.New("failed to compress image using tinify")
}
// Resize and download image
j, _ = json.Marshal(map[string]interface{}{
"resize": map[string]interface{}{
"method": "fit",
"width": defaultCompressionWidth,
"height": defaultCompressionHeight,
},
})
downloadReq, err := http.NewRequest(http.MethodPost, compressedLocation, bytes.NewReader(j))
var imgBuffer bytes.Buffer
err = requests.
URL(compressedLocation).
Client(hc).
Post().
BasicAuth("api", tf.key).
BodyJSON(map[string]interface{}{
"resize": map[string]interface{}{
"method": "fit",
"width": defaultCompressionWidth,
"height": defaultCompressionHeight,
},
}).
ToBytesBuffer(&imgBuffer).
Fetch(context.Background())
if err != nil {
return "", err
}
downloadReq.SetBasicAuth("api", tf.key)
downloadReq.Header.Set(contentType, contenttype.JSON)
downloadResp, err := hc.Do(downloadReq)
if err != nil {
return "", err
}
defer downloadResp.Body.Close()
if downloadResp.StatusCode != http.StatusOK {
return "", fmt.Errorf("tinify failed to resize image, status code %d", downloadResp.StatusCode)
}
tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension)
if err != nil {
return "", err
}
defer func() {
_ = tmpFile.Close()
_ = os.Remove(tmpFile.Name())
}()
if _, err = io.Copy(tmpFile, downloadResp.Body); err != nil {
return "", err
}
fileName, err := getSHA256(tmpFile)
if err != nil {
return "", err
log.Println("Tinify error:", err.Error())
return "", errors.New("failed to compress image using tinify")
}
// Upload compressed file
location, err = upload(fileName+"."+fileExtension, tmpFile)
return
return uploadCompressedFile(fileExtension, &imgBuffer, upload)
}
type cloudflare struct {
}
type cloudflare struct{}
var _ mediaCompression = &cloudflare{}
func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (location string, err error) {
func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (string, error) {
// Check url
_, allowed := urlHasExt(url, "jpg", "jpeg", "png")
if !allowed {
if _, allowed := urlHasExt(url, "jpg", "jpeg", "png"); !allowed {
return "", nil
}
// Force jpeg
fileExtension := "jpeg"
// Compress
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=%d,h=%d/%s", defaultCompressionWidth, defaultCompressionHeight, url), nil)
var imgBuffer bytes.Buffer
err := requests.
URL(fmt.Sprintf("https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=%d,h=%d/%s", defaultCompressionWidth, defaultCompressionHeight, url)).
Client(hc).
Get().
ToBytesBuffer(&imgBuffer).
Fetch(context.Background())
if err != nil {
return "", err
log.Println("Cloudflare error:", err.Error())
return "", errors.New("failed to compress image using cloudflare")
}
resp, err := hc.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("cloudflare failed to compress image, status code %d", resp.StatusCode)
}
tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension)
if err != nil {
return "", err
}
defer func() {
_ = tmpFile.Close()
_ = os.Remove(tmpFile.Name())
}()
if _, err = io.Copy(tmpFile, resp.Body); err != nil {
return "", err
}
fileName, err := getSHA256(tmpFile)
// Upload compressed file
return uploadCompressedFile(fileExtension, &imgBuffer, upload)
}
func uploadCompressedFile(fileExtension string, imgBuffer *bytes.Buffer, upload mediaStorageSaveFunc) (string, error) {
// Create reader from buffer
imgReader := bytes.NewReader(imgBuffer.Bytes())
// Get hash of compressed file
fileName, err := getSHA256(imgReader)
if err != nil {
return "", err
}
// Upload compressed file
location, err = upload(fileName+"."+fileExtension, tmpFile)
return
return upload(fileName+"."+fileExtension, imgReader)
}