Browse Source

Refactor media storage, add support for FTP

master
Jan-Lukas Else 5 months ago
parent
commit
8db544150d
  1. 6
      app.go
  2. 2
      blogroll.go
  3. 21
      config.go
  4. 7
      example-config.yml
  5. 2
      go.mod
  6. 5
      go.sum
  7. 17
      media.go
  8. 32
      mediaCompression.go
  9. 2
      mediaCompression_test.go
  10. 125
      mediaStorage.go
  11. 42
      micropubMedia.go
  12. 4
      paths.go
  13. 5
      paths_test.go

6
app.go

@ -76,8 +76,10 @@ type goBlog struct {
// Markdown
md, absoluteMd goldmark.Markdown
// Media
compressorsInit sync.Once
compressors []mediaCompression
compressorsInit sync.Once
compressors []mediaCompression
mediaStorageInit sync.Once
mediaStorage mediaStorage
// Minify
min minify.Minifier
// Regex Redirects

2
blogroll.go

@ -87,7 +87,7 @@ func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) {
res.Body.Close()
}()
if code := res.StatusCode; code < 200 || 300 <= code {
return nil, fmt.Errorf("opml request not successfull, status code: %d", code)
return nil, fmt.Errorf("opml request not successful, status code: %d", code)
}
o, err := opml.Parse(res.Body)
if err != nil {

21
config.go

@ -185,12 +185,21 @@ type configMicropub struct {
}
type configMicropubMedia struct {
MediaURL string `mapstructure:"mediaUrl"`
BunnyStorageKey string `mapstructure:"bunnyStorageKey"`
BunnyStorageName string `mapstructure:"bunnyStorageName"`
TinifyKey string `mapstructure:"tinifyKey"`
ShortPixelKey string `mapstructure:"shortPixelKey"`
CloudflareCompressionEnabled bool `mapstructure:"cloudflareCompressionEnabled"`
MediaURL string `mapstructure:"mediaUrl"`
// BunnyCDN
BunnyStorageKey string `mapstructure:"bunnyStorageKey"`
BunnyStorageName string `mapstructure:"bunnyStorageName"`
BunnyStorageRegion string `mapstructure:"bunnyStorageRegion"`
// FTP
FTPAddress string `mapstructure:"ftpAddress"`
FTPUser string `mapstructure:"ftpUser"`
FTPPassword string `mapstructure:"ftpPassword"`
// Tinify
TinifyKey string `mapstructure:"tinifyKey"`
// Shortpixel
ShortPixelKey string `mapstructure:"shortPixelKey"`
// Cloudflare
CloudflareCompressionEnabled bool `mapstructure:"cloudflareCompressionEnabled"`
}
type configRegexRedirect struct {

7
example-config.yml

@ -79,10 +79,15 @@ webmention:
micropub:
# Media configuration
mediaStorage:
mediaUrl: https://media.example.com # Define external media URL (instead of /m subpath)
mediaUrl: https://media.example.com # Define external media URL (instead of /m subpath for local files), required for BunnyCDN and FTP
# BunnyCDN storage (optional)
bunnyStorageKey: BUNNY-STORAGE-KEY # Secret key for BunnyCDN storage
bunnyStorageName: storagename # BunnyCDN storage name
bunnyStorageRegion: ny # required if BunnyCDN storage region isn't Falkenstein
# FTP storage (optional)
ftpAddress: ftp.example.com:21 # Host and port for FTP connection
ftpUser: ftpuser # Username of FTP user
ftpPassword: ftppassword # Password of FTP user
# Image compression (optional, you can define no, one or multiple services, disabled when private mode enabled)
shortPixelKey: SHORT-PIXEL-KEY # Secret key for the ShortPixel API
tinifyKey: TINIFY-KEY # Secret key for the Tinify.com API (first fallback)

2
go.mod

@ -27,6 +27,7 @@ require (
github.com/gorilla/handlers v1.5.1
github.com/gorilla/securecookie v1.1.1
github.com/gorilla/sessions v1.2.1
github.com/jlaffaye/ftp v0.0.0-20210307004419-5d4190119067
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9
github.com/kr/text v0.2.0 // indirect
@ -36,6 +37,7 @@ require (
github.com/lopezator/migrator v0.3.0
github.com/mattn/go-sqlite3 v1.14.7
github.com/microcosm-cc/bluemonday v1.0.14
github.com/miekg/dns v1.1.43 // indirect
github.com/mitchellh/go-server-timing v1.0.1
github.com/paulmach/go.geojson v1.4.0
github.com/pquerna/otp v1.3.0

5
go.sum

@ -243,6 +243,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jlaffaye/ftp v0.0.0-20210307004419-5d4190119067 h1:P2S26PMwXl8+ZGuOG3C69LG4be5vHafUayZm9VPw3tU=
github.com/jlaffaye/ftp v0.0.0-20210307004419-5d4190119067/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@ -297,8 +299,9 @@ github.com/mholt/acmez v0.1.3/go.mod h1:8qnn8QA/Ewx8E3ZSsmscqsIjhhpxuy9vqdgbX2ce
github.com/microcosm-cc/bluemonday v1.0.14 h1:Djd+GeTanVeA23todvVC0AO5hsI+vAwQMLTy794Zr5I=
github.com/microcosm-cc/bluemonday v1.0.14/go.mod h1:beubO5lmWoy1tU8niaMyXNriNgROO37H3U/tsrcZsy0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.42 h1:gWGe42RGaIqXQZ+r3WUGEKBEtvPHY2SXo4dqixDNxuY=
github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-server-timing v1.0.1 h1:f00/aIe8T3MrnLhQHu3tSWvnwc5GV/p5eutuu3hF/tE=

17
media.go

@ -1,7 +1,6 @@
package main
import (
"io"
"net/http"
"os"
"path/filepath"
@ -11,22 +10,6 @@ import (
const mediaFilePath = "data/media"
func saveMediaFile(filename string, mediaFile io.Reader) (string, error) {
err := os.MkdirAll(mediaFilePath, 0644)
if err != nil {
return "", err
}
newFile, err := os.Create(filepath.Join(mediaFilePath, filename))
if err != nil {
return "", err
}
_, err = io.Copy(newFile, mediaFile)
if err != nil {
return "", err
}
return "/m/" + filename, nil
}
func (a *goBlog) serveMediaFile(w http.ResponseWriter, r *http.Request) {
f := filepath.Join(mediaFilePath, chi.URLParam(r, "file"))
_, err := os.Stat(f)

32
mediaCompression.go

@ -16,18 +16,7 @@ const defaultCompressionWidth = 2000
const defaultCompressionHeight = 3000
type mediaCompression interface {
compress(url string, save fileUploadFunc, hc httpClient) (location string, err error)
}
type shortpixel struct {
key string
}
type tinify struct {
key string
}
type cloudflare struct {
compress(url string, save mediaStorageSaveFunc, hc httpClient) (location string, err error)
}
func (a *goBlog) compressMediaFile(url string) (location string, err error) {
@ -35,7 +24,7 @@ func (a *goBlog) compressMediaFile(url string) (location string, err error) {
a.compressorsInit.Do(a.initMediaCompressors)
// Try all compressors until success
for _, c := range a.compressors {
location, err = c.compress(url, a.uploadFile, a.httpClient)
location, err = c.compress(url, a.saveMediaFile, a.httpClient)
if location != "" && err == nil {
break
}
@ -60,7 +49,11 @@ func (a *goBlog) initMediaCompressors() {
}
}
func (sp *shortpixel) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) {
type shortpixel struct {
key string
}
func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc httpClient) (location string, err error) {
// Check url
fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png")
if !allowed {
@ -113,7 +106,11 @@ func (sp *shortpixel) compress(url string, upload fileUploadFunc, hc httpClient)
return
}
func (tf *tinify) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) {
type tinify struct {
key string
}
func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc httpClient) (location string, err error) {
// Check url
fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png")
if !allowed {
@ -189,7 +186,10 @@ func (tf *tinify) compress(url string, upload fileUploadFunc, hc httpClient) (lo
return
}
func (cf *cloudflare) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) {
type cloudflare struct {
}
func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc httpClient) (location string, err error) {
// Check url
_, allowed := urlHasExt(url, "jpg", "jpeg", "png")
if !allowed {

2
mediaCompression_test.go

@ -22,7 +22,7 @@ func Test_compress(t *testing.T) {
fakeSha256, err := getSHA256(fakeFile)
require.Nil(t, err)
var uf fileUploadFunc = func(filename string, f io.Reader) (location string, err error) {
var uf mediaStorageSaveFunc = func(filename string, f io.Reader) (location string, err error) {
return "https://example.com/" + filename, nil
}

125
mediaStorage.go

@ -0,0 +1,125 @@
package main
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/jlaffaye/ftp"
)
type mediaStorageSaveFunc func(filename string, file io.Reader) (location string, err error)
func (a *goBlog) saveMediaFile(filename string, f io.Reader) (string, error) {
a.mediaStorageInit.Do(func() {
type initFunc func() mediaStorage
for _, fc := range []initFunc{a.initBunnyCdnMediaStorage, a.initFtpMediaStorage, a.initLocalMediaStorage} {
a.mediaStorage = fc()
if a.mediaStorage != nil {
break
}
}
})
if a.mediaStorage == nil {
return "", errors.New("no media storage configured")
}
loc, err := a.mediaStorage.save(filename, f)
if err != nil {
return "", err
}
return a.getFullAddress(loc), nil
}
type mediaStorage interface {
save(filename string, file io.Reader) (location string, err error)
}
type localMediaStorage struct {
mediaURL string // optional
}
func (a *goBlog) initLocalMediaStorage() mediaStorage {
ms := &localMediaStorage{}
if config := a.cfg.Micropub.MediaStorage; config != nil && config.MediaURL != "" {
ms.mediaURL = config.MediaURL
}
return ms
}
func (l *localMediaStorage) save(filename string, file io.Reader) (location string, err error) {
if err = os.MkdirAll(mediaFilePath, 0644); err != nil {
return "", err
}
newFile, err := os.Create(filepath.Join(mediaFilePath, filename))
if err != nil {
return "", err
}
if _, err = io.Copy(newFile, file); err != nil {
return "", err
}
if l.mediaURL != "" {
return fmt.Sprintf("%s/%s", l.mediaURL, filename), nil
}
return fmt.Sprintf("/m/%s", filename), nil
}
func (a *goBlog) initBunnyCdnMediaStorage() mediaStorage {
config := a.cfg.Micropub.MediaStorage
if config == nil || config.BunnyStorageName == "" || config.BunnyStorageKey == "" || config.MediaURL == "" {
return nil
}
address := "storage.bunnycdn.com:21"
if config.BunnyStorageRegion != "" {
address = fmt.Sprintf("%s.%s", strings.ToLower(config.BunnyStorageRegion), address)
}
return &ftpMediaStorage{
address: address,
user: config.BunnyStorageName,
password: config.BunnyStorageKey,
mediaURL: config.MediaURL,
}
}
type ftpMediaStorage struct {
address string // required
user string // required
password string // required
mediaURL string // required
}
func (a *goBlog) initFtpMediaStorage() mediaStorage {
config := a.cfg.Micropub.MediaStorage
if config == nil || config.FTPAddress == "" || config.FTPUser == "" || config.FTPPassword == "" {
return nil
}
return &ftpMediaStorage{
address: config.FTPAddress,
user: config.FTPUser,
password: config.FTPPassword,
mediaURL: config.MediaURL,
}
}
func (f *ftpMediaStorage) save(filename string, file io.Reader) (location string, err error) {
if f.address == "" || f.user == "" || f.password == "" {
return "", errors.New("missing FTP config")
}
c, err := ftp.Dial(f.address, ftp.DialWithTimeout(5*time.Second))
if err != nil {
return "", err
}
defer func() {
_ = c.Quit()
}()
if err = c.Login(f.user, f.password); err != nil {
return "", err
}
if err = c.Stor(filename, file); err != nil {
return "", err
}
return fmt.Sprintf("%s/%s", f.mediaURL, filename), nil
}

42
micropubMedia.go

@ -1,12 +1,8 @@
package main
import (
"errors"
"fmt"
"io"
"mime"
"net/http"
"net/url"
"path/filepath"
"strings"
@ -55,7 +51,7 @@ func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) {
}
fileName += strings.ToLower(fileExtension)
// Save file
location, err := a.uploadFile(fileName, file)
location, err := a.saveMediaFile(fileName, file)
if err != nil {
a.serveError(w, r, "failed to save original file: "+err.Error(), http.StatusInternalServerError)
return
@ -74,39 +70,3 @@ func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) {
}
http.Redirect(w, r, location, http.StatusCreated)
}
type fileUploadFunc func(filename string, f io.Reader) (location string, err error)
func (a *goBlog) uploadFile(filename string, f io.Reader) (string, error) {
ms := a.cfg.Micropub.MediaStorage
if ms != nil && ms.BunnyStorageKey != "" && ms.BunnyStorageName != "" {
return a.uploadToBunny(filename, f)
}
loc, err := saveMediaFile(filename, f)
if err != nil {
return "", err
}
if ms != nil && ms.MediaURL != "" {
return ms.MediaURL + loc, nil
}
return a.getFullAddress(loc), nil
}
func (a *goBlog) uploadToBunny(filename string, f io.Reader) (location string, err error) {
config := a.cfg.Micropub.MediaStorage
if config == nil || config.BunnyStorageName == "" || config.BunnyStorageKey == "" || config.MediaURL == "" {
return "", errors.New("Bunny storage not completely configured")
}
req, _ := http.NewRequest(http.MethodPut, fmt.Sprintf("https://storage.bunnycdn.com/%s/%s", url.PathEscape(config.BunnyStorageName), url.PathEscape(filename)), f)
req.Header.Add("AccessKey", config.BunnyStorageKey)
resp, err := a.httpClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, resp.Body)
if resp.StatusCode != http.StatusCreated {
return "", errors.New("failed to upload file to BunnyCDN")
}
return config.MediaURL + "/" + filename, nil
}

4
paths.go

@ -35,6 +35,10 @@ func (a *goBlog) getFullAddress(path string) string {
}
func (cfg *configServer) getFullAddress(path string) string {
// Check if it is already an absolute URL
if isAbsoluteURL(path) {
return path
}
// Remove trailing slash
pa := strings.TrimSuffix(cfg.PublicAddress, "/")
// Check if path is root => blank path

5
paths_test.go

@ -3,6 +3,8 @@ package main
import (
"reflect"
"testing"
"github.com/stretchr/testify/assert"
)
func Test_getFullAddress(t *testing.T) {
@ -39,6 +41,9 @@ func Test_getFullAddress(t *testing.T) {
if got := cfg1.getFullAddress(""); !reflect.DeepEqual(got, "https://example.com") {
t.Errorf("Wrong full path, got: %v", got)
}
assert.Equal(t, "https://example.net", cfg1.getFullAddress("https://example.net"))
assert.Equal(t, "https://example.net", cfg2.getFullAddress("https://example.net"))
}
func Test_getRelativeBlogPath(t *testing.T) {

Loading…
Cancel
Save