Add local image compression using VIPS

pull/25/head
Jan-Lukas Else 9 months ago
parent 48679bc1cf
commit a5bdb23979
  1. 4
      Dockerfile
  2. 2
      config.go
  3. 1
      docs/build.md
  4. 1
      docs/usage.md
  5. 5
      example-config.yml
  6. 2
      go.mod
  7. 11
      go.sum
  8. 76
      mediaCompression.go

@ -1,7 +1,7 @@
FROM golang:1.18-alpine3.15 as buildbase
WORKDIR /app
RUN apk add --no-cache git gcc musl-dev
RUN apk add --no-cache git gcc musl-dev vips-dev
RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main sqlite-dev
ENV GOFLAGS="-tags=linux,libsqlite3,sqlite_fts5"
ADD *.go go.mod go.sum /app/
@ -30,7 +30,7 @@ EXPOSE 443
EXPOSE 8080
CMD ["GoBlog"]
HEALTHCHECK --interval=1m --timeout=10s CMD GoBlog healthcheck
RUN apk add --no-cache tzdata tor
RUN apk add --no-cache tzdata tor vips-dev
RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main sqlite-dev
COPY templates/ /app/templates/
COPY --from=build /app/GoBlog /bin/

@ -244,6 +244,8 @@ type configMicropubMedia struct {
TinifyKey string `mapstructure:"tinifyKey"`
// Cloudflare
CloudflareCompressionEnabled bool `mapstructure:"cloudflareCompressionEnabled"`
// Local
LocalCompressionEnabled bool `mapstructure:"localCompressionEnabled"`
}
type configRegexRedirect struct {

@ -32,6 +32,7 @@ Requirements:
- git
- go >= 1.17
- libsqlite3 with FTS5 enabled >= 3.31 (the newer the better)
- libvips >= 8.10 (the newer the better)
Build command:

@ -27,6 +27,7 @@ It is possible to configure multiple compression providers. If one fails, the ne
1. Tinify
2. Cloudflare
3. Local compression using VIPS
## Text-to-Speech

@ -111,8 +111,9 @@ micropub:
ftpUser: ftpuser # Username of FTP user
ftpPassword: ftppassword # Password of FTP user
# Image compression (optional, you can define no, one or multiple services, disabled when private mode enabled)
tinifyKey: TINIFY-KEY # Secret key for the Tinify.com API (first fallback)
cloudflareCompressionEnabled: true # Use Cloudflare's compression as a second fallback
tinifyKey: TINIFY-KEY # Secret key for the Tinify.com API
cloudflareCompressionEnabled: true # Use Cloudflare's compression
localCompressionEnabled: true # Use local compression
# MicroPub parameters (defaults already set, set to overwrite)
# You can set parameters via the UI of your MicroPub editor or via front matter in the content
categoryParam: tags

@ -13,6 +13,7 @@ require (
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
github.com/carlmjohnson/requests v0.22.1
github.com/cretz/bine v0.2.0
github.com/davidbyttow/govips/v2 v2.11.0
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
github.com/dgraph-io/ristretto v0.1.0
github.com/dmulholl/mp3lib v1.0.0
@ -129,6 +130,7 @@ require (
go4.org/mem v0.0.0-20210711025021-927187094b94 // indirect
go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
golang.org/x/image v0.0.0-20220321031419-a8550c1d254a // indirect
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect

@ -94,6 +94,8 @@ github.com/cretz/bine v0.2.0/go.mod h1:WU4o9QR9wWp8AVKtTM1XD5vUHkEqnf2vVSo6dBqbe
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davidbyttow/govips/v2 v2.11.0 h1:eJY+Sgt2LRVh6TFSNMnl5rrFkDfuToG5uE5aLSV1jvM=
github.com/davidbyttow/govips/v2 v2.11.0/go.mod h1:goq38QD8XEMz2aWEeucEZqRxAWsemIN40vbUqfPfTAw=
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI=
@ -370,6 +372,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/paulmach/go.geojson v1.4.0 h1:5x5moCkCtDo5x8af62P9IOAYGQcYHtxz2QJ3x1DoCgY=
github.com/paulmach/go.geojson v1.4.0/go.mod h1:YaKx1hKpWF+T2oj2lFJPsW/t1Q5e1jQI61eoQSTwpIs=
github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
@ -502,6 +505,9 @@ golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl4
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20200927104501-e162460cd6b5/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20220321031419-a8550c1d254a h1:LnH9RNcpPv5Kzi15lXg42lYMPUf0x8CuPv1YnvBWZAg=
golang.org/x/image v0.0.0-20220321031419-a8550c1d254a/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -556,6 +562,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
@ -661,6 +668,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@ -807,7 +815,8 @@ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+Rur
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI=
gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

@ -1,15 +1,18 @@
package main
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
"log"
"math"
"net/http"
"github.com/carlmjohnson/requests"
"github.com/davidbyttow/govips/v2/vips"
"go.goblog.app/app/pkgs/bufferpool"
)
@ -45,6 +48,9 @@ func (a *goBlog) initMediaCompressors() {
if config.CloudflareCompressionEnabled {
a.compressors = append(a.compressors, &cloudflare{})
}
if config.LocalCompressionEnabled {
a.compressors = append(a.compressors, &localMediaCompressor{})
}
}
type tinify struct {
@ -131,6 +137,76 @@ func (*cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http.Cl
return uploadCompressedFile(fileExtension, imgBuffer, upload)
}
type localMediaCompressor struct{}
func (*localMediaCompressor) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (string, error) {
// Check url
fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png")
if !allowed {
return "", nil
}
// Download image
imgBuffer := bufferpool.Get()
defer bufferpool.Put(imgBuffer)
err := requests.
URL(url).
Client(hc).
ToBytesBuffer(imgBuffer).
Fetch(context.Background())
if err != nil {
log.Println("Local compressor error:", err.Error())
return "", errors.New("failed to download image using local compressor")
}
// Compress image using bimg
vips.Startup(&vips.Config{
CollectStats: false,
MaxCacheFiles: 0,
MaxCacheMem: 0,
MaxCacheSize: 0,
ConcurrencyLevel: 1,
})
img, err := vips.NewImageFromReader(imgBuffer)
if err != nil {
log.Println("Local compressor error:", err.Error())
return "", errors.New("failed to compress image using local compressor")
}
// Auto rotate image
err = img.AutoRotate()
if err != nil {
log.Println("Local compressor error:", err.Error())
return "", errors.New("failed to compress image using local compressor")
}
// Get resize ratio to fit in max width and height
ratioHeight := float64(defaultCompressionHeight) / float64(img.Height())
ratioWidth := float64(defaultCompressionWidth) / float64(img.Width())
ratio := math.Min(ratioHeight, ratioWidth)
// Resize image (only downscale)
if ratio < 1 {
err = img.Resize(ratio, vips.KernelAuto)
if err != nil {
log.Println("Local compressor error:", err.Error())
return "", errors.New("failed to compress image using local compressor")
}
}
// Export resized image
ep := vips.NewDefaultExportParams()
ep.StripMetadata = true // Strip metadata
ep.Interlaced = true // Progressive image
ep.TrellisQuant = true // Trellis quantization (JPEG only)
ep.OptimizeCoding = true // Optimize Huffman coding tables (JPEG only)
ep.OptimizeScans = true // Optimize progressive scans (JPEG only)
ep.OvershootDeringing = true // Overshoot deringing (JPEG only)
ep.Compression = 100 // Compression factor
ep.Quality = 75 // Quality factor
compressed, _, err := img.Export(ep)
if err != nil {
log.Println("Local compressor error:", err.Error())
return "", errors.New("failed to compress image using local compressor")
}
// Upload compressed file
return uploadCompressedFile(fileExtension, bytes.NewReader(compressed), upload)
}
func uploadCompressedFile(fileExtension string, r io.Reader, upload mediaStorageSaveFunc) (string, error) {
// Copy file to temporary buffer to generate hash and filename
hash := sha256.New()

Loading…
Cancel
Save