From a5bdb23979424fb8322217765f08d5747718bf12 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Else Date: Thu, 24 Mar 2022 15:19:34 +0100 Subject: [PATCH] Add local image compression using VIPS --- Dockerfile | 4 +-- config.go | 2 ++ docs/build.md | 1 + docs/usage.md | 1 + example-config.yml | 5 +-- go.mod | 2 ++ go.sum | 11 ++++++- mediaCompression.go | 76 +++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 97 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 23c26d4..64dcd84 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.18-alpine3.15 as buildbase WORKDIR /app -RUN apk add --no-cache git gcc musl-dev +RUN apk add --no-cache git gcc musl-dev vips-dev RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main sqlite-dev ENV GOFLAGS="-tags=linux,libsqlite3,sqlite_fts5" ADD *.go go.mod go.sum /app/ @@ -30,7 +30,7 @@ EXPOSE 443 EXPOSE 8080 CMD ["GoBlog"] HEALTHCHECK --interval=1m --timeout=10s CMD GoBlog healthcheck -RUN apk add --no-cache tzdata tor +RUN apk add --no-cache tzdata tor vips-dev RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main sqlite-dev COPY templates/ /app/templates/ COPY --from=build /app/GoBlog /bin/ diff --git a/config.go b/config.go index 98d2407..dd2a9a6 100644 --- a/config.go +++ b/config.go @@ -244,6 +244,8 @@ type configMicropubMedia struct { TinifyKey string `mapstructure:"tinifyKey"` // Cloudflare CloudflareCompressionEnabled bool `mapstructure:"cloudflareCompressionEnabled"` + // Local + LocalCompressionEnabled bool `mapstructure:"localCompressionEnabled"` } type configRegexRedirect struct { diff --git a/docs/build.md b/docs/build.md index ba98b61..2f9ad50 100644 --- a/docs/build.md +++ b/docs/build.md @@ -32,6 +32,7 @@ Requirements: - git - go >= 1.17 - libsqlite3 with FTS5 enabled >= 3.31 (the newer the better) +- libvips >= 8.10 (the newer the better) Build command: diff --git a/docs/usage.md b/docs/usage.md index ccfac81..51890ce 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -27,6 +27,7 @@ It is possible to configure multiple compression providers. If one fails, the ne 1. Tinify 2. Cloudflare +3. Local compression using VIPS ## Text-to-Speech diff --git a/example-config.yml b/example-config.yml index 2fcbafa..abf4474 100644 --- a/example-config.yml +++ b/example-config.yml @@ -111,8 +111,9 @@ micropub: ftpUser: ftpuser # Username of FTP user ftpPassword: ftppassword # Password of FTP user # Image compression (optional, you can define no, one or multiple services, disabled when private mode enabled) - tinifyKey: TINIFY-KEY # Secret key for the Tinify.com API (first fallback) - cloudflareCompressionEnabled: true # Use Cloudflare's compression as a second fallback + tinifyKey: TINIFY-KEY # Secret key for the Tinify.com API + cloudflareCompressionEnabled: true # Use Cloudflare's compression + localCompressionEnabled: true # Use local compression # MicroPub parameters (defaults already set, set to overwrite) # You can set parameters via the UI of your MicroPub editor or via front matter in the content categoryParam: tags diff --git a/go.mod b/go.mod index a4f0030..b2e56c0 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 github.com/carlmjohnson/requests v0.22.1 github.com/cretz/bine v0.2.0 + github.com/davidbyttow/govips/v2 v2.11.0 github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f github.com/dgraph-io/ristretto v0.1.0 github.com/dmulholl/mp3lib v1.0.0 @@ -129,6 +130,7 @@ require ( go4.org/mem v0.0.0-20210711025021-927187094b94 // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect + golang.org/x/image v0.0.0-20220321031419-a8550c1d254a // indirect golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect diff --git a/go.sum b/go.sum index 71b7c5d..395173f 100644 --- a/go.sum +++ b/go.sum @@ -94,6 +94,8 @@ github.com/cretz/bine v0.2.0/go.mod h1:WU4o9QR9wWp8AVKtTM1XD5vUHkEqnf2vVSo6dBqbe github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidbyttow/govips/v2 v2.11.0 h1:eJY+Sgt2LRVh6TFSNMnl5rrFkDfuToG5uE5aLSV1jvM= +github.com/davidbyttow/govips/v2 v2.11.0/go.mod h1:goq38QD8XEMz2aWEeucEZqRxAWsemIN40vbUqfPfTAw= github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M= github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY= github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= @@ -370,6 +372,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/paulmach/go.geojson v1.4.0 h1:5x5moCkCtDo5x8af62P9IOAYGQcYHtxz2QJ3x1DoCgY= github.com/paulmach/go.geojson v1.4.0/go.mod h1:YaKx1hKpWF+T2oj2lFJPsW/t1Q5e1jQI61eoQSTwpIs= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= @@ -502,6 +505,9 @@ golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl4 golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200927104501-e162460cd6b5/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20220321031419-a8550c1d254a h1:LnH9RNcpPv5Kzi15lXg42lYMPUf0x8CuPv1YnvBWZAg= +golang.org/x/image v0.0.0-20220321031419-a8550c1d254a/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -556,6 +562,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -661,6 +668,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -807,7 +815,8 @@ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+Rur google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/mediaCompression.go b/mediaCompression.go index 0e56e08..8d7ecfa 100644 --- a/mediaCompression.go +++ b/mediaCompression.go @@ -1,15 +1,18 @@ package main import ( + "bytes" "context" "crypto/sha256" "errors" "fmt" "io" "log" + "math" "net/http" "github.com/carlmjohnson/requests" + "github.com/davidbyttow/govips/v2/vips" "go.goblog.app/app/pkgs/bufferpool" ) @@ -45,6 +48,9 @@ func (a *goBlog) initMediaCompressors() { if config.CloudflareCompressionEnabled { a.compressors = append(a.compressors, &cloudflare{}) } + if config.LocalCompressionEnabled { + a.compressors = append(a.compressors, &localMediaCompressor{}) + } } type tinify struct { @@ -131,6 +137,76 @@ func (*cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http.Cl return uploadCompressedFile(fileExtension, imgBuffer, upload) } +type localMediaCompressor struct{} + +func (*localMediaCompressor) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (string, error) { + // Check url + fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png") + if !allowed { + return "", nil + } + // Download image + imgBuffer := bufferpool.Get() + defer bufferpool.Put(imgBuffer) + err := requests. + URL(url). + Client(hc). + ToBytesBuffer(imgBuffer). + Fetch(context.Background()) + if err != nil { + log.Println("Local compressor error:", err.Error()) + return "", errors.New("failed to download image using local compressor") + } + // Compress image using bimg + vips.Startup(&vips.Config{ + CollectStats: false, + MaxCacheFiles: 0, + MaxCacheMem: 0, + MaxCacheSize: 0, + ConcurrencyLevel: 1, + }) + img, err := vips.NewImageFromReader(imgBuffer) + if err != nil { + log.Println("Local compressor error:", err.Error()) + return "", errors.New("failed to compress image using local compressor") + } + // Auto rotate image + err = img.AutoRotate() + if err != nil { + log.Println("Local compressor error:", err.Error()) + return "", errors.New("failed to compress image using local compressor") + } + // Get resize ratio to fit in max width and height + ratioHeight := float64(defaultCompressionHeight) / float64(img.Height()) + ratioWidth := float64(defaultCompressionWidth) / float64(img.Width()) + ratio := math.Min(ratioHeight, ratioWidth) + // Resize image (only downscale) + if ratio < 1 { + err = img.Resize(ratio, vips.KernelAuto) + if err != nil { + log.Println("Local compressor error:", err.Error()) + return "", errors.New("failed to compress image using local compressor") + } + } + // Export resized image + ep := vips.NewDefaultExportParams() + ep.StripMetadata = true // Strip metadata + ep.Interlaced = true // Progressive image + ep.TrellisQuant = true // Trellis quantization (JPEG only) + ep.OptimizeCoding = true // Optimize Huffman coding tables (JPEG only) + ep.OptimizeScans = true // Optimize progressive scans (JPEG only) + ep.OvershootDeringing = true // Overshoot deringing (JPEG only) + ep.Compression = 100 // Compression factor + ep.Quality = 75 // Quality factor + compressed, _, err := img.Export(ep) + if err != nil { + log.Println("Local compressor error:", err.Error()) + return "", errors.New("failed to compress image using local compressor") + } + // Upload compressed file + return uploadCompressedFile(fileExtension, bytes.NewReader(compressed), upload) +} + func uploadCompressedFile(fileExtension string, r io.Reader, upload mediaStorageSaveFunc) (string, error) { // Copy file to temporary buffer to generate hash and filename hash := sha256.New()