Refactor media storage, add support for FTP

This commit is contained in:
Jan-Lukas Else 2021-06-23 14:28:51 +02:00
parent f96a06beac
commit 8db544150d
13 changed files with 184 additions and 86 deletions

6
app.go
View File

@ -76,8 +76,10 @@ type goBlog struct {
// Markdown // Markdown
md, absoluteMd goldmark.Markdown md, absoluteMd goldmark.Markdown
// Media // Media
compressorsInit sync.Once compressorsInit sync.Once
compressors []mediaCompression compressors []mediaCompression
mediaStorageInit sync.Once
mediaStorage mediaStorage
// Minify // Minify
min minify.Minifier min minify.Minifier
// Regex Redirects // Regex Redirects

View File

@ -87,7 +87,7 @@ func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) {
res.Body.Close() res.Body.Close()
}() }()
if code := res.StatusCode; code < 200 || 300 <= code { if code := res.StatusCode; code < 200 || 300 <= code {
return nil, fmt.Errorf("opml request not successfull, status code: %d", code) return nil, fmt.Errorf("opml request not successful, status code: %d", code)
} }
o, err := opml.Parse(res.Body) o, err := opml.Parse(res.Body)
if err != nil { if err != nil {

View File

@ -185,12 +185,21 @@ type configMicropub struct {
} }
type configMicropubMedia struct { type configMicropubMedia struct {
MediaURL string `mapstructure:"mediaUrl"` MediaURL string `mapstructure:"mediaUrl"`
BunnyStorageKey string `mapstructure:"bunnyStorageKey"` // BunnyCDN
BunnyStorageName string `mapstructure:"bunnyStorageName"` BunnyStorageKey string `mapstructure:"bunnyStorageKey"`
TinifyKey string `mapstructure:"tinifyKey"` BunnyStorageName string `mapstructure:"bunnyStorageName"`
ShortPixelKey string `mapstructure:"shortPixelKey"` BunnyStorageRegion string `mapstructure:"bunnyStorageRegion"`
CloudflareCompressionEnabled bool `mapstructure:"cloudflareCompressionEnabled"` // FTP
FTPAddress string `mapstructure:"ftpAddress"`
FTPUser string `mapstructure:"ftpUser"`
FTPPassword string `mapstructure:"ftpPassword"`
// Tinify
TinifyKey string `mapstructure:"tinifyKey"`
// Shortpixel
ShortPixelKey string `mapstructure:"shortPixelKey"`
// Cloudflare
CloudflareCompressionEnabled bool `mapstructure:"cloudflareCompressionEnabled"`
} }
type configRegexRedirect struct { type configRegexRedirect struct {

View File

@ -79,10 +79,15 @@ webmention:
micropub: micropub:
# Media configuration # Media configuration
mediaStorage: mediaStorage:
mediaUrl: https://media.example.com # Define external media URL (instead of /m subpath) mediaUrl: https://media.example.com # Define external media URL (instead of /m subpath for local files), required for BunnyCDN and FTP
# BunnyCDN storage (optional) # BunnyCDN storage (optional)
bunnyStorageKey: BUNNY-STORAGE-KEY # Secret key for BunnyCDN storage bunnyStorageKey: BUNNY-STORAGE-KEY # Secret key for BunnyCDN storage
bunnyStorageName: storagename # BunnyCDN storage name bunnyStorageName: storagename # BunnyCDN storage name
bunnyStorageRegion: ny # required if BunnyCDN storage region isn't Falkenstein
# FTP storage (optional)
ftpAddress: ftp.example.com:21 # Host and port for FTP connection
ftpUser: ftpuser # Username of FTP user
ftpPassword: ftppassword # Password of FTP user
# Image compression (optional, you can define no, one or multiple services, disabled when private mode enabled) # Image compression (optional, you can define no, one or multiple services, disabled when private mode enabled)
shortPixelKey: SHORT-PIXEL-KEY # Secret key for the ShortPixel API shortPixelKey: SHORT-PIXEL-KEY # Secret key for the ShortPixel API
tinifyKey: TINIFY-KEY # Secret key for the Tinify.com API (first fallback) tinifyKey: TINIFY-KEY # Secret key for the Tinify.com API (first fallback)

2
go.mod
View File

@ -27,6 +27,7 @@ require (
github.com/gorilla/handlers v1.5.1 github.com/gorilla/handlers v1.5.1
github.com/gorilla/securecookie v1.1.1 github.com/gorilla/securecookie v1.1.1
github.com/gorilla/sessions v1.2.1 github.com/gorilla/sessions v1.2.1
github.com/jlaffaye/ftp v0.0.0-20210307004419-5d4190119067
github.com/jonboulle/clockwork v0.2.2 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9 github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9
github.com/kr/text v0.2.0 // indirect github.com/kr/text v0.2.0 // indirect
@ -36,6 +37,7 @@ require (
github.com/lopezator/migrator v0.3.0 github.com/lopezator/migrator v0.3.0
github.com/mattn/go-sqlite3 v1.14.7 github.com/mattn/go-sqlite3 v1.14.7
github.com/microcosm-cc/bluemonday v1.0.14 github.com/microcosm-cc/bluemonday v1.0.14
github.com/miekg/dns v1.1.43 // indirect
github.com/mitchellh/go-server-timing v1.0.1 github.com/mitchellh/go-server-timing v1.0.1
github.com/paulmach/go.geojson v1.4.0 github.com/paulmach/go.geojson v1.4.0
github.com/pquerna/otp v1.3.0 github.com/pquerna/otp v1.3.0

5
go.sum
View File

@ -243,6 +243,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jlaffaye/ftp v0.0.0-20210307004419-5d4190119067 h1:P2S26PMwXl8+ZGuOG3C69LG4be5vHafUayZm9VPw3tU=
github.com/jlaffaye/ftp v0.0.0-20210307004419-5d4190119067/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@ -297,8 +299,9 @@ github.com/mholt/acmez v0.1.3/go.mod h1:8qnn8QA/Ewx8E3ZSsmscqsIjhhpxuy9vqdgbX2ce
github.com/microcosm-cc/bluemonday v1.0.14 h1:Djd+GeTanVeA23todvVC0AO5hsI+vAwQMLTy794Zr5I= github.com/microcosm-cc/bluemonday v1.0.14 h1:Djd+GeTanVeA23todvVC0AO5hsI+vAwQMLTy794Zr5I=
github.com/microcosm-cc/bluemonday v1.0.14/go.mod h1:beubO5lmWoy1tU8niaMyXNriNgROO37H3U/tsrcZsy0= github.com/microcosm-cc/bluemonday v1.0.14/go.mod h1:beubO5lmWoy1tU8niaMyXNriNgROO37H3U/tsrcZsy0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.42 h1:gWGe42RGaIqXQZ+r3WUGEKBEtvPHY2SXo4dqixDNxuY=
github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-server-timing v1.0.1 h1:f00/aIe8T3MrnLhQHu3tSWvnwc5GV/p5eutuu3hF/tE= github.com/mitchellh/go-server-timing v1.0.1 h1:f00/aIe8T3MrnLhQHu3tSWvnwc5GV/p5eutuu3hF/tE=

View File

@ -1,7 +1,6 @@
package main package main
import ( import (
"io"
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
@ -11,22 +10,6 @@ import (
const mediaFilePath = "data/media" const mediaFilePath = "data/media"
func saveMediaFile(filename string, mediaFile io.Reader) (string, error) {
err := os.MkdirAll(mediaFilePath, 0644)
if err != nil {
return "", err
}
newFile, err := os.Create(filepath.Join(mediaFilePath, filename))
if err != nil {
return "", err
}
_, err = io.Copy(newFile, mediaFile)
if err != nil {
return "", err
}
return "/m/" + filename, nil
}
func (a *goBlog) serveMediaFile(w http.ResponseWriter, r *http.Request) { func (a *goBlog) serveMediaFile(w http.ResponseWriter, r *http.Request) {
f := filepath.Join(mediaFilePath, chi.URLParam(r, "file")) f := filepath.Join(mediaFilePath, chi.URLParam(r, "file"))
_, err := os.Stat(f) _, err := os.Stat(f)

View File

@ -16,18 +16,7 @@ const defaultCompressionWidth = 2000
const defaultCompressionHeight = 3000 const defaultCompressionHeight = 3000
type mediaCompression interface { type mediaCompression interface {
compress(url string, save fileUploadFunc, hc httpClient) (location string, err error) compress(url string, save mediaStorageSaveFunc, hc httpClient) (location string, err error)
}
type shortpixel struct {
key string
}
type tinify struct {
key string
}
type cloudflare struct {
} }
func (a *goBlog) compressMediaFile(url string) (location string, err error) { func (a *goBlog) compressMediaFile(url string) (location string, err error) {
@ -35,7 +24,7 @@ func (a *goBlog) compressMediaFile(url string) (location string, err error) {
a.compressorsInit.Do(a.initMediaCompressors) a.compressorsInit.Do(a.initMediaCompressors)
// Try all compressors until success // Try all compressors until success
for _, c := range a.compressors { for _, c := range a.compressors {
location, err = c.compress(url, a.uploadFile, a.httpClient) location, err = c.compress(url, a.saveMediaFile, a.httpClient)
if location != "" && err == nil { if location != "" && err == nil {
break break
} }
@ -60,7 +49,11 @@ func (a *goBlog) initMediaCompressors() {
} }
} }
func (sp *shortpixel) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) { type shortpixel struct {
key string
}
func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc httpClient) (location string, err error) {
// Check url // Check url
fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png") fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png")
if !allowed { if !allowed {
@ -113,7 +106,11 @@ func (sp *shortpixel) compress(url string, upload fileUploadFunc, hc httpClient)
return return
} }
func (tf *tinify) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) { type tinify struct {
key string
}
func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc httpClient) (location string, err error) {
// Check url // Check url
fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png") fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png")
if !allowed { if !allowed {
@ -189,7 +186,10 @@ func (tf *tinify) compress(url string, upload fileUploadFunc, hc httpClient) (lo
return return
} }
func (cf *cloudflare) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) { type cloudflare struct {
}
func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc httpClient) (location string, err error) {
// Check url // Check url
_, allowed := urlHasExt(url, "jpg", "jpeg", "png") _, allowed := urlHasExt(url, "jpg", "jpeg", "png")
if !allowed { if !allowed {

View File

@ -22,7 +22,7 @@ func Test_compress(t *testing.T) {
fakeSha256, err := getSHA256(fakeFile) fakeSha256, err := getSHA256(fakeFile)
require.Nil(t, err) require.Nil(t, err)
var uf fileUploadFunc = func(filename string, f io.Reader) (location string, err error) { var uf mediaStorageSaveFunc = func(filename string, f io.Reader) (location string, err error) {
return "https://example.com/" + filename, nil return "https://example.com/" + filename, nil
} }

125
mediaStorage.go Normal file
View File

@ -0,0 +1,125 @@
package main
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/jlaffaye/ftp"
)
type mediaStorageSaveFunc func(filename string, file io.Reader) (location string, err error)
func (a *goBlog) saveMediaFile(filename string, f io.Reader) (string, error) {
a.mediaStorageInit.Do(func() {
type initFunc func() mediaStorage
for _, fc := range []initFunc{a.initBunnyCdnMediaStorage, a.initFtpMediaStorage, a.initLocalMediaStorage} {
a.mediaStorage = fc()
if a.mediaStorage != nil {
break
}
}
})
if a.mediaStorage == nil {
return "", errors.New("no media storage configured")
}
loc, err := a.mediaStorage.save(filename, f)
if err != nil {
return "", err
}
return a.getFullAddress(loc), nil
}
type mediaStorage interface {
save(filename string, file io.Reader) (location string, err error)
}
type localMediaStorage struct {
mediaURL string // optional
}
func (a *goBlog) initLocalMediaStorage() mediaStorage {
ms := &localMediaStorage{}
if config := a.cfg.Micropub.MediaStorage; config != nil && config.MediaURL != "" {
ms.mediaURL = config.MediaURL
}
return ms
}
func (l *localMediaStorage) save(filename string, file io.Reader) (location string, err error) {
if err = os.MkdirAll(mediaFilePath, 0644); err != nil {
return "", err
}
newFile, err := os.Create(filepath.Join(mediaFilePath, filename))
if err != nil {
return "", err
}
if _, err = io.Copy(newFile, file); err != nil {
return "", err
}
if l.mediaURL != "" {
return fmt.Sprintf("%s/%s", l.mediaURL, filename), nil
}
return fmt.Sprintf("/m/%s", filename), nil
}
func (a *goBlog) initBunnyCdnMediaStorage() mediaStorage {
config := a.cfg.Micropub.MediaStorage
if config == nil || config.BunnyStorageName == "" || config.BunnyStorageKey == "" || config.MediaURL == "" {
return nil
}
address := "storage.bunnycdn.com:21"
if config.BunnyStorageRegion != "" {
address = fmt.Sprintf("%s.%s", strings.ToLower(config.BunnyStorageRegion), address)
}
return &ftpMediaStorage{
address: address,
user: config.BunnyStorageName,
password: config.BunnyStorageKey,
mediaURL: config.MediaURL,
}
}
type ftpMediaStorage struct {
address string // required
user string // required
password string // required
mediaURL string // required
}
func (a *goBlog) initFtpMediaStorage() mediaStorage {
config := a.cfg.Micropub.MediaStorage
if config == nil || config.FTPAddress == "" || config.FTPUser == "" || config.FTPPassword == "" {
return nil
}
return &ftpMediaStorage{
address: config.FTPAddress,
user: config.FTPUser,
password: config.FTPPassword,
mediaURL: config.MediaURL,
}
}
func (f *ftpMediaStorage) save(filename string, file io.Reader) (location string, err error) {
if f.address == "" || f.user == "" || f.password == "" {
return "", errors.New("missing FTP config")
}
c, err := ftp.Dial(f.address, ftp.DialWithTimeout(5*time.Second))
if err != nil {
return "", err
}
defer func() {
_ = c.Quit()
}()
if err = c.Login(f.user, f.password); err != nil {
return "", err
}
if err = c.Stor(filename, file); err != nil {
return "", err
}
return fmt.Sprintf("%s/%s", f.mediaURL, filename), nil
}

View File

@ -1,12 +1,8 @@
package main package main
import ( import (
"errors"
"fmt"
"io"
"mime" "mime"
"net/http" "net/http"
"net/url"
"path/filepath" "path/filepath"
"strings" "strings"
@ -55,7 +51,7 @@ func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) {
} }
fileName += strings.ToLower(fileExtension) fileName += strings.ToLower(fileExtension)
// Save file // Save file
location, err := a.uploadFile(fileName, file) location, err := a.saveMediaFile(fileName, file)
if err != nil { if err != nil {
a.serveError(w, r, "failed to save original file: "+err.Error(), http.StatusInternalServerError) a.serveError(w, r, "failed to save original file: "+err.Error(), http.StatusInternalServerError)
return return
@ -74,39 +70,3 @@ func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) {
} }
http.Redirect(w, r, location, http.StatusCreated) http.Redirect(w, r, location, http.StatusCreated)
} }
type fileUploadFunc func(filename string, f io.Reader) (location string, err error)
func (a *goBlog) uploadFile(filename string, f io.Reader) (string, error) {
ms := a.cfg.Micropub.MediaStorage
if ms != nil && ms.BunnyStorageKey != "" && ms.BunnyStorageName != "" {
return a.uploadToBunny(filename, f)
}
loc, err := saveMediaFile(filename, f)
if err != nil {
return "", err
}
if ms != nil && ms.MediaURL != "" {
return ms.MediaURL + loc, nil
}
return a.getFullAddress(loc), nil
}
func (a *goBlog) uploadToBunny(filename string, f io.Reader) (location string, err error) {
config := a.cfg.Micropub.MediaStorage
if config == nil || config.BunnyStorageName == "" || config.BunnyStorageKey == "" || config.MediaURL == "" {
return "", errors.New("Bunny storage not completely configured")
}
req, _ := http.NewRequest(http.MethodPut, fmt.Sprintf("https://storage.bunnycdn.com/%s/%s", url.PathEscape(config.BunnyStorageName), url.PathEscape(filename)), f)
req.Header.Add("AccessKey", config.BunnyStorageKey)
resp, err := a.httpClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, resp.Body)
if resp.StatusCode != http.StatusCreated {
return "", errors.New("failed to upload file to BunnyCDN")
}
return config.MediaURL + "/" + filename, nil
}

View File

@ -35,6 +35,10 @@ func (a *goBlog) getFullAddress(path string) string {
} }
func (cfg *configServer) getFullAddress(path string) string { func (cfg *configServer) getFullAddress(path string) string {
// Check if it is already an absolute URL
if isAbsoluteURL(path) {
return path
}
// Remove trailing slash // Remove trailing slash
pa := strings.TrimSuffix(cfg.PublicAddress, "/") pa := strings.TrimSuffix(cfg.PublicAddress, "/")
// Check if path is root => blank path // Check if path is root => blank path

View File

@ -3,6 +3,8 @@ package main
import ( import (
"reflect" "reflect"
"testing" "testing"
"github.com/stretchr/testify/assert"
) )
func Test_getFullAddress(t *testing.T) { func Test_getFullAddress(t *testing.T) {
@ -39,6 +41,9 @@ func Test_getFullAddress(t *testing.T) {
if got := cfg1.getFullAddress(""); !reflect.DeepEqual(got, "https://example.com") { if got := cfg1.getFullAddress(""); !reflect.DeepEqual(got, "https://example.com") {
t.Errorf("Wrong full path, got: %v", got) t.Errorf("Wrong full path, got: %v", got)
} }
assert.Equal(t, "https://example.net", cfg1.getFullAddress("https://example.net"))
assert.Equal(t, "https://example.net", cfg2.getFullAddress("https://example.net"))
} }
func Test_getRelativeBlogPath(t *testing.T) { func Test_getRelativeBlogPath(t *testing.T) {