From adb6a31b3db220e44af1a4ac295a80f9b2d21f76 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Else Date: Wed, 21 Apr 2021 19:47:45 +0200 Subject: [PATCH] Rework tinify implementation --- go.mod | 7 +- go.sum | 13 ++-- mediaCompression.go | 151 ++++++++++++++++++++++++++++++++++++++++++++ micropubMedia.go | 104 ------------------------------ 4 files changed, 160 insertions(+), 115 deletions(-) create mode 100644 mediaCompression.go diff --git a/go.mod b/go.mod index 0b7d57b..691bc54 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module git.jlel.se/jlelse/GoBlog go 1.16 require ( - codeberg.org/jlelse/tinify v0.0.0-20200123222407-7fc9c21822b0 github.com/PuerkitoBio/goquery v1.6.1 github.com/andybalholm/cascadia v1.2.0 // indirect github.com/araddon/dateparse v0.0.0-20210207001429-0eec95c9db7e @@ -56,12 +55,12 @@ require ( github.com/yuin/goldmark-emoji v1.0.1 go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.16.0 // indirect - golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc // indirect + golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect golang.org/x/mod v0.4.1 // indirect - golang.org/x/net v0.0.0-20210420072503-d25e30425868 + golang.org/x/net v0.0.0-20210420210106-798c2154c571 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe // indirect + golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 // indirect golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.62.0 // indirect diff --git a/go.sum b/go.sum index 6b85825..ecbbeab 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,6 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -codeberg.org/jlelse/tinify v0.0.0-20200123222407-7fc9c21822b0 h1:pJX79kTd01NtxEnzhfd4OU2SY9fquKVoO47DUeNKe+8= -codeberg.org/jlelse/tinify v0.0.0-20200123222407-7fc9c21822b0/go.mod h1:X6cM4Sn0aL/4VQ/ku11yxmiV0WIk5XAaYEPHQLQjFFM= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -369,8 +367,8 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc h1:+q90ECDSAQirdykUN6sPEiBXBsp8Csjcca8Oy7bgLTA= -golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b h1:7mWr3k41Qtv8XlltBkDkl8LoP3mpSgBW8BUoxtEdbXg= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -417,8 +415,8 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210420072503-d25e30425868 h1:mHVdVrNGft0Bv5N0WIf3/ujpDOQOe6KxvwlIikPbMr0= -golang.org/x/net v0.0.0-20210420072503-d25e30425868/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210420210106-798c2154c571 h1:Q6Bg8xzKzpFPU4Oi1sBnBTHBwlMsLeEXpu4hYBY8rAg= +golang.org/x/net v0.0.0-20210420210106-798c2154c571/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -454,8 +452,9 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 h1:EjgCl+fVlIaPJSori0ikSz3uV0DOHKWOJFpv1sAAhBM= +golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/mediaCompression.go b/mediaCompression.go new file mode 100644 index 0000000..60dc698 --- /dev/null +++ b/mediaCompression.go @@ -0,0 +1,151 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" +) + +const defaultCompressionWidth = 2000 +const defaultCompressionHeight = 3000 + +func tinify(url string, config *configMicropubMedia) (location string, err error) { + // Check config + if config == nil || config.TinifyKey == "" { + return "", errors.New("service Tinify not configured") + } + // Check url + fileExtension, allowed := compressionIsSupported(url, "jpg", "jpeg", "png") + if !allowed { + return "", nil + } + // Compress + j, _ := json.Marshal(map[string]interface{}{ + "source": map[string]interface{}{ + "url": url, + }, + }) + req, err := http.NewRequest(http.MethodPost, "https://api.tinify.com/shrink", bytes.NewReader(j)) + if err != nil { + return "", err + } + req.SetBasicAuth("api", config.TinifyKey) + req.Header.Set(contentType, contentTypeJSON) + resp, err := appHttpClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + if resp.StatusCode != http.StatusCreated { + return "", fmt.Errorf("failed to compress image, status code %d", resp.StatusCode) + } + compressedLocation := resp.Header.Get("Location") + if compressedLocation == "" { + return "", errors.New("tinify didn't return compressed location") + } + // Resize and download image + j, _ = json.Marshal(map[string]interface{}{ + "resize": map[string]interface{}{ + "method": "fit", + "width": defaultCompressionWidth, + "height": defaultCompressionHeight, + }, + }) + downloadReq, err := http.NewRequest(http.MethodPost, compressedLocation, bytes.NewReader(j)) + if err != nil { + return "", err + } + downloadReq.SetBasicAuth("api", config.TinifyKey) + downloadReq.Header.Set(contentType, contentTypeJSON) + downloadResp, err := appHttpClient.Do(downloadReq) + if err != nil { + return "", err + } + defer downloadResp.Body.Close() + if downloadResp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, downloadResp.Body) + return "", fmt.Errorf("tinify failed to resize image, status code %d", downloadResp.StatusCode) + } + tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension) + if err != nil { + _, _ = io.Copy(io.Discard, downloadResp.Body) + return "", err + } + defer func() { + _ = tmpFile.Close() + _ = os.Remove(tmpFile.Name()) + }() + if _, err = io.Copy(tmpFile, downloadResp.Body); err != nil { + _, _ = io.Copy(io.Discard, downloadResp.Body) + return "", err + } + fileName, err := getSHA256(tmpFile) + if err != nil { + return "", err + } + // Upload compressed file + location, err = uploadFile(fileName+"."+fileExtension, tmpFile) + return +} + +func shortPixel(url string, config *configMicropubMedia) (location string, err error) { + // Check config + if config == nil || config.ShortPixelKey == "" { + return "", errors.New("service ShortPixel not configured") + } + // Check url + fileExtension, allowed := compressionIsSupported(url, "jpg", "jpeg", "png") + if !allowed { + return "", nil + } + // Compress + j, _ := json.Marshal(map[string]interface{}{ + "key": config.ShortPixelKey, + "plugin_version": "GB001", + "lossy": 1, + "resize": 3, + "resize_width": defaultCompressionWidth, + "resize_height": defaultCompressionHeight, + "cmyk2rgb": 1, + "keep_exif": 0, + "url": url, + }) + req, err := http.NewRequest(http.MethodPut, "https://api.shortpixel.com/v2/reducer-sync.php", bytes.NewReader(j)) + if err != nil { + return "", err + } + resp, err := appHttpClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) + return "", fmt.Errorf("shortpixel failed to compress image, status code %d", resp.StatusCode) + } + tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension) + if err != nil { + _, _ = io.Copy(io.Discard, resp.Body) + return "", err + } + defer func() { + _ = tmpFile.Close() + _ = os.Remove(tmpFile.Name()) + }() + if _, err = io.Copy(tmpFile, resp.Body); err != nil { + _, _ = io.Copy(io.Discard, resp.Body) + return "", err + } + fileName, err := getSHA256(tmpFile) + if err != nil { + return "", err + } + // Upload compressed file + location, err = uploadFile(fileName+"."+fileExtension, tmpFile) + return +} diff --git a/micropubMedia.go b/micropubMedia.go index 5c534e5..a307a98 100644 --- a/micropubMedia.go +++ b/micropubMedia.go @@ -1,21 +1,16 @@ package main import ( - "bytes" "crypto/sha256" - "encoding/json" "errors" "fmt" "io" "mime" "net/http" "net/url" - "os" "path/filepath" "sort" "strings" - - tfgo "codeberg.org/jlelse/tinify" ) const micropubMediaSubPath = "/media" @@ -128,105 +123,6 @@ func uploadToBunny(filename string, f io.Reader, config *configMicropubMedia) (l return config.MediaURL + "/" + filename, nil } -func tinify(url string, config *configMicropubMedia) (location string, err error) { - // Check config - if config == nil || config.TinifyKey == "" { - return "", errors.New("Tinify not configured") - } - // Check url - fileExtension, allowed := compressionIsSupported(url, "jpg", "jpeg", "png") - if !allowed { - return "", nil - } - // Compress - tfgo.SetKey(config.TinifyKey) - s, err := tfgo.FromUrl(url) - if err != nil { - return "", err - } - if err = s.Resize(&tfgo.ResizeOption{ - Method: tfgo.ResizeMethodScale, - Width: 2000, - }); err != nil { - return "", err - } - tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension) - if err != nil { - return "", err - } - defer func() { - _ = tmpFile.Close() - _ = os.Remove(tmpFile.Name()) - }() - if err = s.ToFile(tmpFile.Name()); err != nil { - return "", err - } - fileName, err := getSHA256(tmpFile) - if err != nil { - return "", err - } - // Upload compressed file - location, err = uploadFile(fileName+"."+fileExtension, tmpFile) - return -} - -func shortPixel(url string, config *configMicropubMedia) (location string, err error) { - // Check config - if config == nil || config.ShortPixelKey == "" { - return "", errors.New("ShortPixel not configured") - } - // Check url - fileExtension, allowed := compressionIsSupported(url, "jpg", "jpeg", "png") - if !allowed { - return "", nil - } - // Compress - j, _ := json.Marshal(map[string]interface{}{ - "key": config.ShortPixelKey, - "plugin_version": "GB001", - "lossy": 1, - "resize": 3, - "resize_width": 2000, - "resize_height": 3000, - "cmyk2rgb": 1, - "keep_exif": 0, - "url": url, - }) - req, err := http.NewRequest(http.MethodPut, "https://api.shortpixel.com/v2/reducer-sync.php", bytes.NewReader(j)) - if err != nil { - return "", err - } - resp, err := appHttpClient.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - _, _ = io.Copy(io.Discard, resp.Body) - return "", fmt.Errorf("failed to compress image, status code %d", resp.StatusCode) - } - tmpFile, err := os.CreateTemp("", "tiny-*."+fileExtension) - if err != nil { - _, _ = io.Copy(io.Discard, resp.Body) - return "", err - } - defer func() { - _ = tmpFile.Close() - _ = os.Remove(tmpFile.Name()) - }() - if _, err = io.Copy(tmpFile, resp.Body); err != nil { - _, _ = io.Copy(io.Discard, resp.Body) - return "", err - } - fileName, err := getSHA256(tmpFile) - if err != nil { - return "", err - } - // Upload compressed file - location, err = uploadFile(fileName+"."+fileExtension, tmpFile) - return -} - func compressionIsSupported(url string, allowed ...string) (string, bool) { spliced := strings.Split(url, ".") ext := spliced[len(spliced)-1]