Refactor media compression

app.go

@@ -75,6 +75,9 @@ type goBlog struct {
 	logf *rotatelogs.RotateLogs
 	// Markdown
 	md, absoluteMd goldmark.Markdown
+	// Media
+	compressorsInit sync.Once
+	compressors     []mediaCompression
 	// Minify
 	min minify.Minifier
 	// Regex Redirects

httpClient_test.go

@@ -1,35 +1,43 @@
 package main
 
 import (
-	"io"
 	"net/http"
-	"strings"
+	"net/http/httptest"
 )
 
 type fakeHttpClient struct {
-	req *http.Request
-	res *http.Response
-	err error
+	req     *http.Request
+	res     *http.Response
+	handler http.Handler
 }
 
 func (c *fakeHttpClient) Do(req *http.Request) (*http.Response, error) {
+	if c.handler == nil {
+		return nil, nil
+	}
+	rec := httptest.NewRecorder()
+	c.handler.ServeHTTP(rec, req)
 	c.req = req
-	return c.res, c.err
+	c.res = rec.Result()
+	return c.res, nil
 }
 
 func (c *fakeHttpClient) clean() {
 	c.req = nil
-	c.err = nil
 	c.res = nil
+	c.handler = nil
 }
 
-func (c *fakeHttpClient) setFakeResponse(statusCode int, body string, err error) {
+func (c *fakeHttpClient) setHandler(handler http.Handler) {
 	c.clean()
-	c.err = err
-	c.res = &http.Response{
-		StatusCode: statusCode,
-		Body:       io.NopCloser(strings.NewReader(body)),
-	}
+	c.handler = handler
+}
+
+func (c *fakeHttpClient) setFakeResponse(statusCode int, body string) {
+	c.setHandler(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		rw.WriteHeader(statusCode)
+		rw.Write([]byte(body))
+	}))
 }
 
 func getFakeHTTPClient() *fakeHttpClient {

mediaCompression.go

@@ -15,99 +15,60 @@ import (
 const defaultCompressionWidth = 2000
 const defaultCompressionHeight = 3000
 
-func (a *goBlog) tinify(url string, config *configMicropubMedia) (location string, err error) {
-	// Check config
-	if config == nil || config.TinifyKey == "" {
-		return "", errors.New("service Tinify not configured")
-	}
-	// Check url
-	fileExtension, allowed := compressionIsSupported(url, "jpg", "jpeg", "png")
-	if !allowed {
-		return "", nil
-	}
-	// Compress
-	j, _ := json.Marshal(map[string]interface{}{
-		"source": map[string]interface{}{
-			"url": url,
-		},
-	})
-	req, err := http.NewRequest(http.MethodPost, "https://api.tinify.com/shrink", bytes.NewReader(j))
-	if err != nil {
-		return "", err
-	}
-	req.SetBasicAuth("api", config.TinifyKey)
-	req.Header.Set(contentType, contenttype.JSON)
-	resp, err := a.httpClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	_, _ = io.Copy(io.Discard, resp.Body)
-	if resp.StatusCode != http.StatusCreated {
-		return "", fmt.Errorf("failed to compress image, status code %d", resp.StatusCode)
-	}
-	compressedLocation := resp.Header.Get("Location")
-	if compressedLocation == "" {
-		return "", errors.New("tinify didn't return compressed location")
-	}
-	// Resize and download image
-	j, _ = json.Marshal(map[string]interface{}{
-		"resize": map[string]interface{}{
-			"method": "fit",
-			"width":  defaultCompressionWidth,
-			"height": defaultCompressionHeight,
-		},
-	})
-	downloadReq, err := http.NewRequest(http.MethodPost, compressedLocation, bytes.NewReader(j))
-	if err != nil {
-		return "", err
-	}
-	downloadReq.SetBasicAuth("api", config.TinifyKey)
-	downloadReq.Header.Set(contentType, contenttype.JSON)
-	downloadResp, err := a.httpClient.Do(downloadReq)
-	if err != nil {
-		return "", err
-	}
-	defer downloadResp.Body.Close()
-	if downloadResp.StatusCode != http.StatusOK {
-		_, _ = io.Copy(io.Discard, downloadResp.Body)
-		return "", fmt.Errorf("tinify failed to resize image, status code %d", downloadResp.StatusCode)
-	}
-	tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension)
-	if err != nil {
-		_, _ = io.Copy(io.Discard, downloadResp.Body)
-		return "", err
-	}
-	defer func() {
-		_ = tmpFile.Close()
-		_ = os.Remove(tmpFile.Name())
-	}()
-	if _, err = io.Copy(tmpFile, downloadResp.Body); err != nil {
-		_, _ = io.Copy(io.Discard, downloadResp.Body)
-		return "", err
-	}
-	fileName, err := getSHA256(tmpFile)
-	if err != nil {
-		return "", err
-	}
-	// Upload compressed file
-	location, err = a.uploadFile(fileName+"."+fileExtension, tmpFile)
-	return
+type mediaCompression interface {
+	compress(url string, save fileUploadFunc, hc httpClient) (location string, err error)
 }
 
-func (a *goBlog) shortPixel(url string, config *configMicropubMedia) (location string, err error) {
-	// Check config
-	if config == nil || config.ShortPixelKey == "" {
-		return "", errors.New("service ShortPixel not configured")
+type shortpixel struct {
+	key string
+}
+
+type tinify struct {
+	key string
+}
+
+type cloudflare struct {
+}
+
+func (a *goBlog) compressMediaFile(url string) (location string, err error) {
+	// Init compressors
+	a.compressorsInit.Do(a.initMediaCompressors)
+	// Try all compressors until success
+	for _, c := range a.compressors {
+		location, err = c.compress(url, a.uploadFile, a.httpClient)
+		if location != "" && err == nil {
+			break
+		}
 	}
+	// Return result
+	return location, err
+}
+
+func (a *goBlog) initMediaCompressors() {
+	config := a.cfg.Micropub.MediaStorage
+	if config == nil {
+		return
+	}
+	if key := config.ShortPixelKey; key != "" {
+		a.compressors = append(a.compressors, &shortpixel{key})
+	}
+	if key := config.TinifyKey; key != "" {
+		a.compressors = append(a.compressors, &tinify{key})
+	}
+	if config.CloudflareCompressionEnabled {
+		a.compressors = append(a.compressors, &cloudflare{})
+	}
+}
+
+func (sp *shortpixel) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) {
 	// Check url
-	fileExtension, allowed := compressionIsSupported(url, "jpg", "jpeg", "png")
+	fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png")
 	if !allowed {
 		return "", nil
 	}
 	// Compress
 	j, _ := json.Marshal(map[string]interface{}{
-		"key":            config.ShortPixelKey,
+		"key":            sp.key,
 		"plugin_version": "GB001",
 		"lossy":          1,
 		"resize":         3,
@@ -121,7 +82,7 @@ func (a *goBlog) shortPixel(url string, config *configMicropubMedia) (location s
 	if err != nil {
 		return "", err
 	}
-	resp, err := a.httpClient.Do(req)
+	resp, err := hc.Do(req)
 	if err != nil {
 		return "", err
 	}
@@ -148,13 +109,89 @@ func (a *goBlog) shortPixel(url string, config *configMicropubMedia) (location s
 		return "", err
 	}
 	// Upload compressed file
-	location, err = a.uploadFile(fileName+"."+fileExtension, tmpFile)
+	location, err = upload(fileName+"."+fileExtension, tmpFile)
 	return
 }
 
-func (a *goBlog) cloudflare(url string) (location string, err error) {
+func (tf *tinify) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) {
 	// Check url
-	_, allowed := compressionIsSupported(url, "jpg", "jpeg", "png")
+	fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png")
+	if !allowed {
+		return "", nil
+	}
+	// Compress
+	j, _ := json.Marshal(map[string]interface{}{
+		"source": map[string]interface{}{
+			"url": url,
+		},
+	})
+	req, err := http.NewRequest(http.MethodPost, "https://api.tinify.com/shrink", bytes.NewReader(j))
+	if err != nil {
+		return "", err
+	}
+	req.SetBasicAuth("api", tf.key)
+	req.Header.Set(contentType, contenttype.JSON)
+	resp, err := hc.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	_, _ = io.Copy(io.Discard, resp.Body)
+	if resp.StatusCode != http.StatusCreated {
+		return "", fmt.Errorf("failed to compress image, status code %d", resp.StatusCode)
+	}
+	compressedLocation := resp.Header.Get("Location")
+	if compressedLocation == "" {
+		return "", errors.New("tinify didn't return compressed location")
+	}
+	// Resize and download image
+	j, _ = json.Marshal(map[string]interface{}{
+		"resize": map[string]interface{}{
+			"method": "fit",
+			"width":  defaultCompressionWidth,
+			"height": defaultCompressionHeight,
+		},
+	})
+	downloadReq, err := http.NewRequest(http.MethodPost, compressedLocation, bytes.NewReader(j))
+	if err != nil {
+		return "", err
+	}
+	downloadReq.SetBasicAuth("api", tf.key)
+	downloadReq.Header.Set(contentType, contenttype.JSON)
+	downloadResp, err := hc.Do(downloadReq)
+	if err != nil {
+		return "", err
+	}
+	defer downloadResp.Body.Close()
+	if downloadResp.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, downloadResp.Body)
+		return "", fmt.Errorf("tinify failed to resize image, status code %d", downloadResp.StatusCode)
+	}
+	tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension)
+	if err != nil {
+		_, _ = io.Copy(io.Discard, downloadResp.Body)
+		return "", err
+	}
+	defer func() {
+		_ = tmpFile.Close()
+		_ = os.Remove(tmpFile.Name())
+	}()
+	if _, err = io.Copy(tmpFile, downloadResp.Body); err != nil {
+		_, _ = io.Copy(io.Discard, downloadResp.Body)
+		return "", err
+	}
+	fileName, err := getSHA256(tmpFile)
+	if err != nil {
+		return "", err
+	}
+	// Upload compressed file
+	location, err = upload(fileName+"."+fileExtension, tmpFile)
+	return
+}
+
+func (cf *cloudflare) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) {
+	// Check url
+	_, allowed := urlHasExt(url, "jpg", "jpeg", "png")
 	if !allowed {
 		return "", nil
 	}
@@ -165,7 +202,7 @@ func (a *goBlog) cloudflare(url string) (location string, err error) {
 	if err != nil {
 		return "", err
 	}
-	resp, err := a.httpClient.Do(req)
+	resp, err := hc.Do(req)
 	if err != nil {
 		return "", err
 	}
@@ -192,6 +229,6 @@ func (a *goBlog) cloudflare(url string) (location string, err error) {
 		return "", err
 	}
 	// Upload compressed file
-	location, err = a.uploadFile(fileName+"."+fileExtension, tmpFile)
+	location, err = upload(fileName+"."+fileExtension, tmpFile)
 	return
 }

mediaCompression_test.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_compress(t *testing.T) {
+	fakeFileContent := "Test"
+	fakeFileName := filepath.Join(t.TempDir(), "test.jpg")
+	err := os.WriteFile(fakeFileName, []byte(fakeFileContent), 0777)
+	require.Nil(t, err)
+	fakeFile, err := os.Open(fakeFileName)
+	require.Nil(t, err)
+	fakeSha256, err := getSHA256(fakeFile)
+	require.Nil(t, err)
+
+	var uf fileUploadFunc = func(filename string, f io.Reader) (location string, err error) {
+		return "https://example.com/" + filename, nil
+	}
+
+	t.Run("Cloudflare", func(t *testing.T) {
+		fakeClient := getFakeHTTPClient()
+		fakeClient.setHandler(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			assert.Equal(t, "https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=2000,h=3000/https://example.com/original.jpg", r.URL.String())
+
+			rw.WriteHeader(http.StatusOK)
+			rw.Write([]byte(fakeFileContent))
+		}))
+
+		cf := &cloudflare{}
+		res, err := cf.compress("https://example.com/original.jpg", uf, fakeClient)
+
+		assert.Nil(t, err)
+		assert.Equal(t, "https://example.com/"+fakeSha256+".jpeg", res)
+	})
+
+	t.Run("Shortpixel", func(t *testing.T) {
+		fakeClient := getFakeHTTPClient()
+		fakeClient.setHandler(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			assert.Equal(t, "https://api.shortpixel.com/v2/reducer-sync.php", r.URL.String())
+
+			requestBody, _ := io.ReadAll(r.Body)
+			defer r.Body.Close()
+
+			var requestJson map[string]interface{}
+			err = json.Unmarshal(requestBody, &requestJson)
+			require.Nil(t, err)
+			require.NotNil(t, requestJson)
+
+			assert.Equal(t, "testkey", requestJson["key"])
+			assert.Equal(t, "https://example.com/original.jpg", requestJson["url"])
+
+			rw.WriteHeader(http.StatusOK)
+			rw.Write([]byte(fakeFileContent))
+		}))
+
+		cf := &shortpixel{"testkey"}
+		res, err := cf.compress("https://example.com/original.jpg", uf, fakeClient)
+
+		assert.Nil(t, err)
+		assert.Equal(t, "https://example.com/"+fakeSha256+".jpg", res)
+	})
+
+}

micropubMedia.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io"
@@ -9,7 +8,6 @@ import (
 	"net/http"
 	"net/url"
 	"path/filepath"
-	"sort"
 	"strings"
 
 	"git.jlel.se/jlelse/GoBlog/pkgs/contenttype"
@@ -63,46 +61,22 @@ func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	// Try to compress file (only when not in private mode)
-	if pm := a.cfg.PrivateMode; !(pm != nil && pm.Enabled) {
-		serveCompressionError := func(ce error) {
-			a.serveError(w, r, "failed to compress file: "+ce.Error(), http.StatusInternalServerError)
+	if pm := a.cfg.PrivateMode; pm == nil || !pm.Enabled {
+		compressedLocation, compressionErr := a.compressMediaFile(location)
+		if compressionErr != nil {
+			a.serveError(w, r, "failed to compress file: "+compressionErr.Error(), http.StatusInternalServerError)
+			return
 		}
-		var compressedLocation string
-		var compressionErr error
-		if ms := a.cfg.Micropub.MediaStorage; ms != nil {
-			// Default ShortPixel
-			if ms.ShortPixelKey != "" {
-				compressedLocation, compressionErr = a.shortPixel(location, ms)
-			}
-			if compressionErr != nil {
-				serveCompressionError(compressionErr)
-				return
-			}
-			// Fallback Tinify
-			if compressedLocation == "" && ms.TinifyKey != "" {
-				compressedLocation, compressionErr = a.tinify(location, ms)
-			}
-			if compressionErr != nil {
-				serveCompressionError(compressionErr)
-				return
-			}
-			// Fallback Cloudflare
-			if compressedLocation == "" && ms.CloudflareCompressionEnabled {
-				compressedLocation, compressionErr = a.cloudflare(location)
-			}
-			if compressionErr != nil {
-				serveCompressionError(compressionErr)
-				return
-			}
-			// Overwrite location
-			if compressedLocation != "" {
-				location = compressedLocation
-			}
+		// Overwrite location
+		if compressedLocation != "" {
+			location = compressedLocation
 		}
 	}
 	http.Redirect(w, r, location, http.StatusCreated)
 }
 
+type fileUploadFunc func(filename string, f io.Reader) (location string, err error)
+
 func (a *goBlog) uploadFile(filename string, f io.Reader) (string, error) {
 	ms := a.cfg.Micropub.MediaStorage
 	if ms != nil && ms.BunnyStorageKey != "" && ms.BunnyStorageName != "" {
@@ -136,27 +110,3 @@ func (a *goBlog) uploadToBunny(filename string, f io.Reader) (location string, e
 	}
 	return config.MediaURL + "/" + filename, nil
 }
-
-func compressionIsSupported(url string, allowed ...string) (string, bool) {
-	spliced := strings.Split(url, ".")
-	ext := spliced[len(spliced)-1]
-	sort.Strings(allowed)
-	if i := sort.SearchStrings(allowed, strings.ToLower(ext)); i >= len(allowed) || allowed[i] != strings.ToLower(ext) {
-		return ext, false
-	}
-	return ext, true
-}
-
-func getSHA256(file io.ReadSeeker) (filename string, err error) {
-	if _, err = file.Seek(0, 0); err != nil {
-		return "", err
-	}
-	h := sha256.New()
-	if _, err = io.Copy(h, file); err != nil {
-		return "", err
-	}
-	if _, err = file.Seek(0, 0); err != nil {
-		return "", err
-	}
-	return fmt.Sprintf("%x", h.Sum(nil)), nil
-}

telegram_test.go

@@ -83,13 +83,12 @@ func Test_configTelegram_send(t *testing.T) {
 		httpClient: fakeClient,
 	}
 
-	fakeClient.setFakeResponse(200, "", nil)
+	fakeClient.setFakeResponse(200, "")
 
 	err := app.send(tg, "Message", "HTML")
 	assert.Nil(t, err)
 
 	assert.NotNil(t, fakeClient.req)
-	assert.Nil(t, fakeClient.err)
 	assert.Equal(t, http.MethodPost, fakeClient.req.Method)
 	assert.Equal(t, "https://api.telegram.org/botbottoken/sendMessage?chat_id=chatid&parse_mode=HTML&text=Message", fakeClient.req.URL.String())
 }
@@ -110,7 +109,7 @@ func Test_telegram(t *testing.T) {
 	t.Run("Send post to Telegram", func(t *testing.T) {
 		fakeClient := getFakeHTTPClient()
 
-		fakeClient.setFakeResponse(200, "", nil)
+		fakeClient.setFakeResponse(200, "")
 
 		app := &goBlog{
 			pPostHooks: []postHookFunc{},
@@ -157,7 +156,7 @@ func Test_telegram(t *testing.T) {
 	t.Run("Telegram disabled", func(t *testing.T) {
 		fakeClient := getFakeHTTPClient()
 
-		fakeClient.setFakeResponse(200, "", nil)
+		fakeClient.setFakeResponse(200, "")
 
 		app := &goBlog{
 			pPostHooks: []postHookFunc{},

utils.go

@@ -1,10 +1,13 @@
 package main
 
 import (
+	"crypto/sha256"
+	"fmt"
 	"html/template"
 	"io"
 	"net/http"
 	"net/url"
+	"path"
 	"sort"
 	"strings"
 	"time"
@@ -191,3 +194,35 @@ func charCount(s string) (count int) {
 func wrapStringAsHTML(s string) template.HTML {
 	return template.HTML(s)
 }
+
+// Check if url has allowed file extension
+func urlHasExt(rawUrl string, allowed ...string) (ext string, has bool) {
+	u, err := url.Parse(rawUrl)
+	if err != nil {
+		return "", false
+	}
+	ext = strings.ToLower(path.Ext(u.Path))
+	if ext == "" {
+		return "", false
+	}
+	ext = ext[1:]
+	allowed = funk.Map(allowed, func(str string) string {
+		return strings.ToLower(str)
+	}).([]string)
+	return ext, funk.ContainsString(allowed, strings.ToLower(ext))
+}
+
+// Get SHA-256 hash of file
+func getSHA256(file io.ReadSeeker) (filename string, err error) {
+	if _, err = file.Seek(0, 0); err != nil {
+		return "", err
+	}
+	h := sha256.New()
+	if _, err = io.Copy(h, file); err != nil {
+		return "", err
+	}
+	if _, err = file.Seek(0, 0); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}

utils_test.go

@@ -83,3 +83,16 @@ func Test_allLinksFromHTMLString(t *testing.T) {
 		t.Errorf("Wrong result, got: %v", result)
 	}
 }
+
+func Test_urlHasExt(t *testing.T) {
+	t.Run("Simple", func(t *testing.T) {
+		ext, res := urlHasExt("https://example.com/test.jpg", "png", "jpg", "webp")
+		assert.True(t, res)
+		assert.Equal(t, "jpg", ext)
+	})
+	t.Run("Strange case", func(t *testing.T) {
+		ext, res := urlHasExt("https://example.com/test.jpG", "PnG", "JPg", "WEBP")
+		assert.True(t, res)
+		assert.Equal(t, "jpg", ext)
+	})
+}