diff --git a/app.go b/app.go index 59744d9..c7f775c 100644 --- a/app.go +++ b/app.go @@ -75,6 +75,9 @@ type goBlog struct { logf *rotatelogs.RotateLogs // Markdown md, absoluteMd goldmark.Markdown + // Media + compressorsInit sync.Once + compressors []mediaCompression // Minify min minify.Minifier // Regex Redirects diff --git a/httpClient_test.go b/httpClient_test.go index fa0e7fb..cb1ec05 100644 --- a/httpClient_test.go +++ b/httpClient_test.go @@ -1,35 +1,43 @@ package main import ( - "io" "net/http" - "strings" + "net/http/httptest" ) type fakeHttpClient struct { - req *http.Request - res *http.Response - err error + req *http.Request + res *http.Response + handler http.Handler } func (c *fakeHttpClient) Do(req *http.Request) (*http.Response, error) { + if c.handler == nil { + return nil, nil + } + rec := httptest.NewRecorder() + c.handler.ServeHTTP(rec, req) c.req = req - return c.res, c.err + c.res = rec.Result() + return c.res, nil } func (c *fakeHttpClient) clean() { c.req = nil - c.err = nil c.res = nil + c.handler = nil } -func (c *fakeHttpClient) setFakeResponse(statusCode int, body string, err error) { +func (c *fakeHttpClient) setHandler(handler http.Handler) { c.clean() - c.err = err - c.res = &http.Response{ - StatusCode: statusCode, - Body: io.NopCloser(strings.NewReader(body)), - } + c.handler = handler +} + +func (c *fakeHttpClient) setFakeResponse(statusCode int, body string) { + c.setHandler(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + rw.WriteHeader(statusCode) + rw.Write([]byte(body)) + })) } func getFakeHTTPClient() *fakeHttpClient { diff --git a/mediaCompression.go b/mediaCompression.go index 81c2370..c3ec6a3 100644 --- a/mediaCompression.go +++ b/mediaCompression.go @@ -15,99 +15,60 @@ import ( const defaultCompressionWidth = 2000 const defaultCompressionHeight = 3000 -func (a *goBlog) tinify(url string, config *configMicropubMedia) (location string, err error) { - // Check config - if config == nil || config.TinifyKey == "" { - return "", errors.New("service Tinify not configured") - } - // Check url - fileExtension, allowed := compressionIsSupported(url, "jpg", "jpeg", "png") - if !allowed { - return "", nil - } - // Compress - j, _ := json.Marshal(map[string]interface{}{ - "source": map[string]interface{}{ - "url": url, - }, - }) - req, err := http.NewRequest(http.MethodPost, "https://api.tinify.com/shrink", bytes.NewReader(j)) - if err != nil { - return "", err - } - req.SetBasicAuth("api", config.TinifyKey) - req.Header.Set(contentType, contenttype.JSON) - resp, err := a.httpClient.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - _, _ = io.Copy(io.Discard, resp.Body) - if resp.StatusCode != http.StatusCreated { - return "", fmt.Errorf("failed to compress image, status code %d", resp.StatusCode) - } - compressedLocation := resp.Header.Get("Location") - if compressedLocation == "" { - return "", errors.New("tinify didn't return compressed location") - } - // Resize and download image - j, _ = json.Marshal(map[string]interface{}{ - "resize": map[string]interface{}{ - "method": "fit", - "width": defaultCompressionWidth, - "height": defaultCompressionHeight, - }, - }) - downloadReq, err := http.NewRequest(http.MethodPost, compressedLocation, bytes.NewReader(j)) - if err != nil { - return "", err - } - downloadReq.SetBasicAuth("api", config.TinifyKey) - downloadReq.Header.Set(contentType, contenttype.JSON) - downloadResp, err := a.httpClient.Do(downloadReq) - if err != nil { - return "", err - } - defer downloadResp.Body.Close() - if downloadResp.StatusCode != http.StatusOK { - _, _ = io.Copy(io.Discard, downloadResp.Body) - return "", fmt.Errorf("tinify failed to resize image, status code %d", downloadResp.StatusCode) - } - tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension) - if err != nil { - _, _ = io.Copy(io.Discard, downloadResp.Body) - return "", err - } - defer func() { - _ = tmpFile.Close() - _ = os.Remove(tmpFile.Name()) - }() - if _, err = io.Copy(tmpFile, downloadResp.Body); err != nil { - _, _ = io.Copy(io.Discard, downloadResp.Body) - return "", err - } - fileName, err := getSHA256(tmpFile) - if err != nil { - return "", err - } - // Upload compressed file - location, err = a.uploadFile(fileName+"."+fileExtension, tmpFile) - return +type mediaCompression interface { + compress(url string, save fileUploadFunc, hc httpClient) (location string, err error) } -func (a *goBlog) shortPixel(url string, config *configMicropubMedia) (location string, err error) { - // Check config - if config == nil || config.ShortPixelKey == "" { - return "", errors.New("service ShortPixel not configured") +type shortpixel struct { + key string +} + +type tinify struct { + key string +} + +type cloudflare struct { +} + +func (a *goBlog) compressMediaFile(url string) (location string, err error) { + // Init compressors + a.compressorsInit.Do(a.initMediaCompressors) + // Try all compressors until success + for _, c := range a.compressors { + location, err = c.compress(url, a.uploadFile, a.httpClient) + if location != "" && err == nil { + break + } } + // Return result + return location, err +} + +func (a *goBlog) initMediaCompressors() { + config := a.cfg.Micropub.MediaStorage + if config == nil { + return + } + if key := config.ShortPixelKey; key != "" { + a.compressors = append(a.compressors, &shortpixel{key}) + } + if key := config.TinifyKey; key != "" { + a.compressors = append(a.compressors, &tinify{key}) + } + if config.CloudflareCompressionEnabled { + a.compressors = append(a.compressors, &cloudflare{}) + } +} + +func (sp *shortpixel) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) { // Check url - fileExtension, allowed := compressionIsSupported(url, "jpg", "jpeg", "png") + fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png") if !allowed { return "", nil } // Compress j, _ := json.Marshal(map[string]interface{}{ - "key": config.ShortPixelKey, + "key": sp.key, "plugin_version": "GB001", "lossy": 1, "resize": 3, @@ -121,7 +82,7 @@ func (a *goBlog) shortPixel(url string, config *configMicropubMedia) (location s if err != nil { return "", err } - resp, err := a.httpClient.Do(req) + resp, err := hc.Do(req) if err != nil { return "", err } @@ -148,13 +109,89 @@ func (a *goBlog) shortPixel(url string, config *configMicropubMedia) (location s return "", err } // Upload compressed file - location, err = a.uploadFile(fileName+"."+fileExtension, tmpFile) + location, err = upload(fileName+"."+fileExtension, tmpFile) return } -func (a *goBlog) cloudflare(url string) (location string, err error) { +func (tf *tinify) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) { // Check url - _, allowed := compressionIsSupported(url, "jpg", "jpeg", "png") + fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png") + if !allowed { + return "", nil + } + // Compress + j, _ := json.Marshal(map[string]interface{}{ + "source": map[string]interface{}{ + "url": url, + }, + }) + req, err := http.NewRequest(http.MethodPost, "https://api.tinify.com/shrink", bytes.NewReader(j)) + if err != nil { + return "", err + } + req.SetBasicAuth("api", tf.key) + req.Header.Set(contentType, contenttype.JSON) + resp, err := hc.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + if resp.StatusCode != http.StatusCreated { + return "", fmt.Errorf("failed to compress image, status code %d", resp.StatusCode) + } + compressedLocation := resp.Header.Get("Location") + if compressedLocation == "" { + return "", errors.New("tinify didn't return compressed location") + } + // Resize and download image + j, _ = json.Marshal(map[string]interface{}{ + "resize": map[string]interface{}{ + "method": "fit", + "width": defaultCompressionWidth, + "height": defaultCompressionHeight, + }, + }) + downloadReq, err := http.NewRequest(http.MethodPost, compressedLocation, bytes.NewReader(j)) + if err != nil { + return "", err + } + downloadReq.SetBasicAuth("api", tf.key) + downloadReq.Header.Set(contentType, contenttype.JSON) + downloadResp, err := hc.Do(downloadReq) + if err != nil { + return "", err + } + defer downloadResp.Body.Close() + if downloadResp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, downloadResp.Body) + return "", fmt.Errorf("tinify failed to resize image, status code %d", downloadResp.StatusCode) + } + tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension) + if err != nil { + _, _ = io.Copy(io.Discard, downloadResp.Body) + return "", err + } + defer func() { + _ = tmpFile.Close() + _ = os.Remove(tmpFile.Name()) + }() + if _, err = io.Copy(tmpFile, downloadResp.Body); err != nil { + _, _ = io.Copy(io.Discard, downloadResp.Body) + return "", err + } + fileName, err := getSHA256(tmpFile) + if err != nil { + return "", err + } + // Upload compressed file + location, err = upload(fileName+"."+fileExtension, tmpFile) + return +} + +func (cf *cloudflare) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) { + // Check url + _, allowed := urlHasExt(url, "jpg", "jpeg", "png") if !allowed { return "", nil } @@ -165,7 +202,7 @@ func (a *goBlog) cloudflare(url string) (location string, err error) { if err != nil { return "", err } - resp, err := a.httpClient.Do(req) + resp, err := hc.Do(req) if err != nil { return "", err } @@ -192,6 +229,6 @@ func (a *goBlog) cloudflare(url string) (location string, err error) { return "", err } // Upload compressed file - location, err = a.uploadFile(fileName+"."+fileExtension, tmpFile) + location, err = upload(fileName+"."+fileExtension, tmpFile) return } diff --git a/mediaCompression_test.go b/mediaCompression_test.go new file mode 100644 index 0000000..e9e57aa --- /dev/null +++ b/mediaCompression_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_compress(t *testing.T) { + fakeFileContent := "Test" + fakeFileName := filepath.Join(t.TempDir(), "test.jpg") + err := os.WriteFile(fakeFileName, []byte(fakeFileContent), 0777) + require.Nil(t, err) + fakeFile, err := os.Open(fakeFileName) + require.Nil(t, err) + fakeSha256, err := getSHA256(fakeFile) + require.Nil(t, err) + + var uf fileUploadFunc = func(filename string, f io.Reader) (location string, err error) { + return "https://example.com/" + filename, nil + } + + t.Run("Cloudflare", func(t *testing.T) { + fakeClient := getFakeHTTPClient() + fakeClient.setHandler(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + assert.Equal(t, "https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=2000,h=3000/https://example.com/original.jpg", r.URL.String()) + + rw.WriteHeader(http.StatusOK) + rw.Write([]byte(fakeFileContent)) + })) + + cf := &cloudflare{} + res, err := cf.compress("https://example.com/original.jpg", uf, fakeClient) + + assert.Nil(t, err) + assert.Equal(t, "https://example.com/"+fakeSha256+".jpeg", res) + }) + + t.Run("Shortpixel", func(t *testing.T) { + fakeClient := getFakeHTTPClient() + fakeClient.setHandler(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + assert.Equal(t, "https://api.shortpixel.com/v2/reducer-sync.php", r.URL.String()) + + requestBody, _ := io.ReadAll(r.Body) + defer r.Body.Close() + + var requestJson map[string]interface{} + err = json.Unmarshal(requestBody, &requestJson) + require.Nil(t, err) + require.NotNil(t, requestJson) + + assert.Equal(t, "testkey", requestJson["key"]) + assert.Equal(t, "https://example.com/original.jpg", requestJson["url"]) + + rw.WriteHeader(http.StatusOK) + rw.Write([]byte(fakeFileContent)) + })) + + cf := &shortpixel{"testkey"} + res, err := cf.compress("https://example.com/original.jpg", uf, fakeClient) + + assert.Nil(t, err) + assert.Equal(t, "https://example.com/"+fakeSha256+".jpg", res) + }) + +} diff --git a/micropubMedia.go b/micropubMedia.go index 089d349..de8fe36 100644 --- a/micropubMedia.go +++ b/micropubMedia.go @@ -1,7 +1,6 @@ package main import ( - "crypto/sha256" "errors" "fmt" "io" @@ -9,7 +8,6 @@ import ( "net/http" "net/url" "path/filepath" - "sort" "strings" "git.jlel.se/jlelse/GoBlog/pkgs/contenttype" @@ -63,46 +61,22 @@ func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) { return } // Try to compress file (only when not in private mode) - if pm := a.cfg.PrivateMode; !(pm != nil && pm.Enabled) { - serveCompressionError := func(ce error) { - a.serveError(w, r, "failed to compress file: "+ce.Error(), http.StatusInternalServerError) + if pm := a.cfg.PrivateMode; pm == nil || !pm.Enabled { + compressedLocation, compressionErr := a.compressMediaFile(location) + if compressionErr != nil { + a.serveError(w, r, "failed to compress file: "+compressionErr.Error(), http.StatusInternalServerError) + return } - var compressedLocation string - var compressionErr error - if ms := a.cfg.Micropub.MediaStorage; ms != nil { - // Default ShortPixel - if ms.ShortPixelKey != "" { - compressedLocation, compressionErr = a.shortPixel(location, ms) - } - if compressionErr != nil { - serveCompressionError(compressionErr) - return - } - // Fallback Tinify - if compressedLocation == "" && ms.TinifyKey != "" { - compressedLocation, compressionErr = a.tinify(location, ms) - } - if compressionErr != nil { - serveCompressionError(compressionErr) - return - } - // Fallback Cloudflare - if compressedLocation == "" && ms.CloudflareCompressionEnabled { - compressedLocation, compressionErr = a.cloudflare(location) - } - if compressionErr != nil { - serveCompressionError(compressionErr) - return - } - // Overwrite location - if compressedLocation != "" { - location = compressedLocation - } + // Overwrite location + if compressedLocation != "" { + location = compressedLocation } } http.Redirect(w, r, location, http.StatusCreated) } +type fileUploadFunc func(filename string, f io.Reader) (location string, err error) + func (a *goBlog) uploadFile(filename string, f io.Reader) (string, error) { ms := a.cfg.Micropub.MediaStorage if ms != nil && ms.BunnyStorageKey != "" && ms.BunnyStorageName != "" { @@ -136,27 +110,3 @@ func (a *goBlog) uploadToBunny(filename string, f io.Reader) (location string, e } return config.MediaURL + "/" + filename, nil } - -func compressionIsSupported(url string, allowed ...string) (string, bool) { - spliced := strings.Split(url, ".") - ext := spliced[len(spliced)-1] - sort.Strings(allowed) - if i := sort.SearchStrings(allowed, strings.ToLower(ext)); i >= len(allowed) || allowed[i] != strings.ToLower(ext) { - return ext, false - } - return ext, true -} - -func getSHA256(file io.ReadSeeker) (filename string, err error) { - if _, err = file.Seek(0, 0); err != nil { - return "", err - } - h := sha256.New() - if _, err = io.Copy(h, file); err != nil { - return "", err - } - if _, err = file.Seek(0, 0); err != nil { - return "", err - } - return fmt.Sprintf("%x", h.Sum(nil)), nil -} diff --git a/telegram_test.go b/telegram_test.go index d1b7dbc..5af50bc 100644 --- a/telegram_test.go +++ b/telegram_test.go @@ -83,13 +83,12 @@ func Test_configTelegram_send(t *testing.T) { httpClient: fakeClient, } - fakeClient.setFakeResponse(200, "", nil) + fakeClient.setFakeResponse(200, "") err := app.send(tg, "Message", "HTML") assert.Nil(t, err) assert.NotNil(t, fakeClient.req) - assert.Nil(t, fakeClient.err) assert.Equal(t, http.MethodPost, fakeClient.req.Method) assert.Equal(t, "https://api.telegram.org/botbottoken/sendMessage?chat_id=chatid&parse_mode=HTML&text=Message", fakeClient.req.URL.String()) } @@ -110,7 +109,7 @@ func Test_telegram(t *testing.T) { t.Run("Send post to Telegram", func(t *testing.T) { fakeClient := getFakeHTTPClient() - fakeClient.setFakeResponse(200, "", nil) + fakeClient.setFakeResponse(200, "") app := &goBlog{ pPostHooks: []postHookFunc{}, @@ -157,7 +156,7 @@ func Test_telegram(t *testing.T) { t.Run("Telegram disabled", func(t *testing.T) { fakeClient := getFakeHTTPClient() - fakeClient.setFakeResponse(200, "", nil) + fakeClient.setFakeResponse(200, "") app := &goBlog{ pPostHooks: []postHookFunc{}, diff --git a/utils.go b/utils.go index ba6f345..9ecd098 100644 --- a/utils.go +++ b/utils.go @@ -1,10 +1,13 @@ package main import ( + "crypto/sha256" + "fmt" "html/template" "io" "net/http" "net/url" + "path" "sort" "strings" "time" @@ -191,3 +194,35 @@ func charCount(s string) (count int) { func wrapStringAsHTML(s string) template.HTML { return template.HTML(s) } + +// Check if url has allowed file extension +func urlHasExt(rawUrl string, allowed ...string) (ext string, has bool) { + u, err := url.Parse(rawUrl) + if err != nil { + return "", false + } + ext = strings.ToLower(path.Ext(u.Path)) + if ext == "" { + return "", false + } + ext = ext[1:] + allowed = funk.Map(allowed, func(str string) string { + return strings.ToLower(str) + }).([]string) + return ext, funk.ContainsString(allowed, strings.ToLower(ext)) +} + +// Get SHA-256 hash of file +func getSHA256(file io.ReadSeeker) (filename string, err error) { + if _, err = file.Seek(0, 0); err != nil { + return "", err + } + h := sha256.New() + if _, err = io.Copy(h, file); err != nil { + return "", err + } + if _, err = file.Seek(0, 0); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} diff --git a/utils_test.go b/utils_test.go index bc7fbd0..8c559ca 100644 --- a/utils_test.go +++ b/utils_test.go @@ -83,3 +83,16 @@ func Test_allLinksFromHTMLString(t *testing.T) { t.Errorf("Wrong result, got: %v", result) } } + +func Test_urlHasExt(t *testing.T) { + t.Run("Simple", func(t *testing.T) { + ext, res := urlHasExt("https://example.com/test.jpg", "png", "jpg", "webp") + assert.True(t, res) + assert.Equal(t, "jpg", ext) + }) + t.Run("Strange case", func(t *testing.T) { + ext, res := urlHasExt("https://example.com/test.jpG", "PnG", "JPg", "WEBP") + assert.True(t, res) + assert.Equal(t, "jpg", ext) + }) +}