diff --git a/app.go b/app.go index c7f775c..5196915 100644 --- a/app.go +++ b/app.go @@ -76,8 +76,10 @@ type goBlog struct { // Markdown md, absoluteMd goldmark.Markdown // Media - compressorsInit sync.Once - compressors []mediaCompression + compressorsInit sync.Once + compressors []mediaCompression + mediaStorageInit sync.Once + mediaStorage mediaStorage // Minify min minify.Minifier // Regex Redirects diff --git a/blogroll.go b/blogroll.go index bcdc78e..5847f3d 100644 --- a/blogroll.go +++ b/blogroll.go @@ -87,7 +87,7 @@ func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) { res.Body.Close() }() if code := res.StatusCode; code < 200 || 300 <= code { - return nil, fmt.Errorf("opml request not successfull, status code: %d", code) + return nil, fmt.Errorf("opml request not successful, status code: %d", code) } o, err := opml.Parse(res.Body) if err != nil { diff --git a/config.go b/config.go index dea3dc0..f42f11e 100644 --- a/config.go +++ b/config.go @@ -185,12 +185,21 @@ type configMicropub struct { } type configMicropubMedia struct { - MediaURL string `mapstructure:"mediaUrl"` - BunnyStorageKey string `mapstructure:"bunnyStorageKey"` - BunnyStorageName string `mapstructure:"bunnyStorageName"` - TinifyKey string `mapstructure:"tinifyKey"` - ShortPixelKey string `mapstructure:"shortPixelKey"` - CloudflareCompressionEnabled bool `mapstructure:"cloudflareCompressionEnabled"` + MediaURL string `mapstructure:"mediaUrl"` + // BunnyCDN + BunnyStorageKey string `mapstructure:"bunnyStorageKey"` + BunnyStorageName string `mapstructure:"bunnyStorageName"` + BunnyStorageRegion string `mapstructure:"bunnyStorageRegion"` + // FTP + FTPAddress string `mapstructure:"ftpAddress"` + FTPUser string `mapstructure:"ftpUser"` + FTPPassword string `mapstructure:"ftpPassword"` + // Tinify + TinifyKey string `mapstructure:"tinifyKey"` + // Shortpixel + ShortPixelKey string `mapstructure:"shortPixelKey"` + // Cloudflare + CloudflareCompressionEnabled bool `mapstructure:"cloudflareCompressionEnabled"` } type configRegexRedirect struct { diff --git a/example-config.yml b/example-config.yml index dff0363..23c2eef 100644 --- a/example-config.yml +++ b/example-config.yml @@ -79,10 +79,15 @@ webmention: micropub: # Media configuration mediaStorage: - mediaUrl: https://media.example.com # Define external media URL (instead of /m subpath) + mediaUrl: https://media.example.com # Define external media URL (instead of /m subpath for local files), required for BunnyCDN and FTP # BunnyCDN storage (optional) bunnyStorageKey: BUNNY-STORAGE-KEY # Secret key for BunnyCDN storage bunnyStorageName: storagename # BunnyCDN storage name + bunnyStorageRegion: ny # required if BunnyCDN storage region isn't Falkenstein + # FTP storage (optional) + ftpAddress: ftp.example.com:21 # Host and port for FTP connection + ftpUser: ftpuser # Username of FTP user + ftpPassword: ftppassword # Password of FTP user # Image compression (optional, you can define no, one or multiple services, disabled when private mode enabled) shortPixelKey: SHORT-PIXEL-KEY # Secret key for the ShortPixel API tinifyKey: TINIFY-KEY # Secret key for the Tinify.com API (first fallback) diff --git a/go.mod b/go.mod index 15bcb4f..a94dd9a 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/gorilla/handlers v1.5.1 github.com/gorilla/securecookie v1.1.1 github.com/gorilla/sessions v1.2.1 + github.com/jlaffaye/ftp v0.0.0-20210307004419-5d4190119067 github.com/jonboulle/clockwork v0.2.2 // indirect github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9 github.com/kr/text v0.2.0 // indirect @@ -36,6 +37,7 @@ require ( github.com/lopezator/migrator v0.3.0 github.com/mattn/go-sqlite3 v1.14.7 github.com/microcosm-cc/bluemonday v1.0.14 + github.com/miekg/dns v1.1.43 // indirect github.com/mitchellh/go-server-timing v1.0.1 github.com/paulmach/go.geojson v1.4.0 github.com/pquerna/otp v1.3.0 diff --git a/go.sum b/go.sum index b77cbc7..030c801 100644 --- a/go.sum +++ b/go.sum @@ -243,6 +243,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jlaffaye/ftp v0.0.0-20210307004419-5d4190119067 h1:P2S26PMwXl8+ZGuOG3C69LG4be5vHafUayZm9VPw3tU= +github.com/jlaffaye/ftp v0.0.0-20210307004419-5d4190119067/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -297,8 +299,9 @@ github.com/mholt/acmez v0.1.3/go.mod h1:8qnn8QA/Ewx8E3ZSsmscqsIjhhpxuy9vqdgbX2ce github.com/microcosm-cc/bluemonday v1.0.14 h1:Djd+GeTanVeA23todvVC0AO5hsI+vAwQMLTy794Zr5I= github.com/microcosm-cc/bluemonday v1.0.14/go.mod h1:beubO5lmWoy1tU8niaMyXNriNgROO37H3U/tsrcZsy0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.42 h1:gWGe42RGaIqXQZ+r3WUGEKBEtvPHY2SXo4dqixDNxuY= github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= +github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-server-timing v1.0.1 h1:f00/aIe8T3MrnLhQHu3tSWvnwc5GV/p5eutuu3hF/tE= diff --git a/media.go b/media.go index fb6cbb0..0b99a80 100644 --- a/media.go +++ b/media.go @@ -1,7 +1,6 @@ package main import ( - "io" "net/http" "os" "path/filepath" @@ -11,22 +10,6 @@ import ( const mediaFilePath = "data/media" -func saveMediaFile(filename string, mediaFile io.Reader) (string, error) { - err := os.MkdirAll(mediaFilePath, 0644) - if err != nil { - return "", err - } - newFile, err := os.Create(filepath.Join(mediaFilePath, filename)) - if err != nil { - return "", err - } - _, err = io.Copy(newFile, mediaFile) - if err != nil { - return "", err - } - return "/m/" + filename, nil -} - func (a *goBlog) serveMediaFile(w http.ResponseWriter, r *http.Request) { f := filepath.Join(mediaFilePath, chi.URLParam(r, "file")) _, err := os.Stat(f) diff --git a/mediaCompression.go b/mediaCompression.go index c3ec6a3..c06100b 100644 --- a/mediaCompression.go +++ b/mediaCompression.go @@ -16,18 +16,7 @@ const defaultCompressionWidth = 2000 const defaultCompressionHeight = 3000 type mediaCompression interface { - compress(url string, save fileUploadFunc, hc httpClient) (location string, err error) -} - -type shortpixel struct { - key string -} - -type tinify struct { - key string -} - -type cloudflare struct { + compress(url string, save mediaStorageSaveFunc, hc httpClient) (location string, err error) } func (a *goBlog) compressMediaFile(url string) (location string, err error) { @@ -35,7 +24,7 @@ func (a *goBlog) compressMediaFile(url string) (location string, err error) { a.compressorsInit.Do(a.initMediaCompressors) // Try all compressors until success for _, c := range a.compressors { - location, err = c.compress(url, a.uploadFile, a.httpClient) + location, err = c.compress(url, a.saveMediaFile, a.httpClient) if location != "" && err == nil { break } @@ -60,7 +49,11 @@ func (a *goBlog) initMediaCompressors() { } } -func (sp *shortpixel) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) { +type shortpixel struct { + key string +} + +func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc httpClient) (location string, err error) { // Check url fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png") if !allowed { @@ -113,7 +106,11 @@ func (sp *shortpixel) compress(url string, upload fileUploadFunc, hc httpClient) return } -func (tf *tinify) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) { +type tinify struct { + key string +} + +func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc httpClient) (location string, err error) { // Check url fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png") if !allowed { @@ -189,7 +186,10 @@ func (tf *tinify) compress(url string, upload fileUploadFunc, hc httpClient) (lo return } -func (cf *cloudflare) compress(url string, upload fileUploadFunc, hc httpClient) (location string, err error) { +type cloudflare struct { +} + +func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc httpClient) (location string, err error) { // Check url _, allowed := urlHasExt(url, "jpg", "jpeg", "png") if !allowed { diff --git a/mediaCompression_test.go b/mediaCompression_test.go index e9e57aa..3e359e8 100644 --- a/mediaCompression_test.go +++ b/mediaCompression_test.go @@ -22,7 +22,7 @@ func Test_compress(t *testing.T) { fakeSha256, err := getSHA256(fakeFile) require.Nil(t, err) - var uf fileUploadFunc = func(filename string, f io.Reader) (location string, err error) { + var uf mediaStorageSaveFunc = func(filename string, f io.Reader) (location string, err error) { return "https://example.com/" + filename, nil } diff --git a/mediaStorage.go b/mediaStorage.go new file mode 100644 index 0000000..d0d2dea --- /dev/null +++ b/mediaStorage.go @@ -0,0 +1,125 @@ +package main + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/jlaffaye/ftp" +) + +type mediaStorageSaveFunc func(filename string, file io.Reader) (location string, err error) + +func (a *goBlog) saveMediaFile(filename string, f io.Reader) (string, error) { + a.mediaStorageInit.Do(func() { + type initFunc func() mediaStorage + for _, fc := range []initFunc{a.initBunnyCdnMediaStorage, a.initFtpMediaStorage, a.initLocalMediaStorage} { + a.mediaStorage = fc() + if a.mediaStorage != nil { + break + } + } + }) + if a.mediaStorage == nil { + return "", errors.New("no media storage configured") + } + loc, err := a.mediaStorage.save(filename, f) + if err != nil { + return "", err + } + return a.getFullAddress(loc), nil +} + +type mediaStorage interface { + save(filename string, file io.Reader) (location string, err error) +} + +type localMediaStorage struct { + mediaURL string // optional +} + +func (a *goBlog) initLocalMediaStorage() mediaStorage { + ms := &localMediaStorage{} + if config := a.cfg.Micropub.MediaStorage; config != nil && config.MediaURL != "" { + ms.mediaURL = config.MediaURL + } + return ms +} + +func (l *localMediaStorage) save(filename string, file io.Reader) (location string, err error) { + if err = os.MkdirAll(mediaFilePath, 0644); err != nil { + return "", err + } + newFile, err := os.Create(filepath.Join(mediaFilePath, filename)) + if err != nil { + return "", err + } + if _, err = io.Copy(newFile, file); err != nil { + return "", err + } + if l.mediaURL != "" { + return fmt.Sprintf("%s/%s", l.mediaURL, filename), nil + } + return fmt.Sprintf("/m/%s", filename), nil +} + +func (a *goBlog) initBunnyCdnMediaStorage() mediaStorage { + config := a.cfg.Micropub.MediaStorage + if config == nil || config.BunnyStorageName == "" || config.BunnyStorageKey == "" || config.MediaURL == "" { + return nil + } + address := "storage.bunnycdn.com:21" + if config.BunnyStorageRegion != "" { + address = fmt.Sprintf("%s.%s", strings.ToLower(config.BunnyStorageRegion), address) + } + return &ftpMediaStorage{ + address: address, + user: config.BunnyStorageName, + password: config.BunnyStorageKey, + mediaURL: config.MediaURL, + } +} + +type ftpMediaStorage struct { + address string // required + user string // required + password string // required + mediaURL string // required +} + +func (a *goBlog) initFtpMediaStorage() mediaStorage { + config := a.cfg.Micropub.MediaStorage + if config == nil || config.FTPAddress == "" || config.FTPUser == "" || config.FTPPassword == "" { + return nil + } + return &ftpMediaStorage{ + address: config.FTPAddress, + user: config.FTPUser, + password: config.FTPPassword, + mediaURL: config.MediaURL, + } +} + +func (f *ftpMediaStorage) save(filename string, file io.Reader) (location string, err error) { + if f.address == "" || f.user == "" || f.password == "" { + return "", errors.New("missing FTP config") + } + c, err := ftp.Dial(f.address, ftp.DialWithTimeout(5*time.Second)) + if err != nil { + return "", err + } + defer func() { + _ = c.Quit() + }() + if err = c.Login(f.user, f.password); err != nil { + return "", err + } + if err = c.Stor(filename, file); err != nil { + return "", err + } + return fmt.Sprintf("%s/%s", f.mediaURL, filename), nil +} diff --git a/micropubMedia.go b/micropubMedia.go index de8fe36..f1cdbcf 100644 --- a/micropubMedia.go +++ b/micropubMedia.go @@ -1,12 +1,8 @@ package main import ( - "errors" - "fmt" - "io" "mime" "net/http" - "net/url" "path/filepath" "strings" @@ -55,7 +51,7 @@ func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) { } fileName += strings.ToLower(fileExtension) // Save file - location, err := a.uploadFile(fileName, file) + location, err := a.saveMediaFile(fileName, file) if err != nil { a.serveError(w, r, "failed to save original file: "+err.Error(), http.StatusInternalServerError) return @@ -74,39 +70,3 @@ func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) { } http.Redirect(w, r, location, http.StatusCreated) } - -type fileUploadFunc func(filename string, f io.Reader) (location string, err error) - -func (a *goBlog) uploadFile(filename string, f io.Reader) (string, error) { - ms := a.cfg.Micropub.MediaStorage - if ms != nil && ms.BunnyStorageKey != "" && ms.BunnyStorageName != "" { - return a.uploadToBunny(filename, f) - } - loc, err := saveMediaFile(filename, f) - if err != nil { - return "", err - } - if ms != nil && ms.MediaURL != "" { - return ms.MediaURL + loc, nil - } - return a.getFullAddress(loc), nil -} - -func (a *goBlog) uploadToBunny(filename string, f io.Reader) (location string, err error) { - config := a.cfg.Micropub.MediaStorage - if config == nil || config.BunnyStorageName == "" || config.BunnyStorageKey == "" || config.MediaURL == "" { - return "", errors.New("Bunny storage not completely configured") - } - req, _ := http.NewRequest(http.MethodPut, fmt.Sprintf("https://storage.bunnycdn.com/%s/%s", url.PathEscape(config.BunnyStorageName), url.PathEscape(filename)), f) - req.Header.Add("AccessKey", config.BunnyStorageKey) - resp, err := a.httpClient.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - _, _ = io.Copy(io.Discard, resp.Body) - if resp.StatusCode != http.StatusCreated { - return "", errors.New("failed to upload file to BunnyCDN") - } - return config.MediaURL + "/" + filename, nil -} diff --git a/paths.go b/paths.go index d6bcd0b..3896d76 100644 --- a/paths.go +++ b/paths.go @@ -35,6 +35,10 @@ func (a *goBlog) getFullAddress(path string) string { } func (cfg *configServer) getFullAddress(path string) string { + // Check if it is already an absolute URL + if isAbsoluteURL(path) { + return path + } // Remove trailing slash pa := strings.TrimSuffix(cfg.PublicAddress, "/") // Check if path is root => blank path diff --git a/paths_test.go b/paths_test.go index 7e0df89..445f98a 100644 --- a/paths_test.go +++ b/paths_test.go @@ -3,6 +3,8 @@ package main import ( "reflect" "testing" + + "github.com/stretchr/testify/assert" ) func Test_getFullAddress(t *testing.T) { @@ -39,6 +41,9 @@ func Test_getFullAddress(t *testing.T) { if got := cfg1.getFullAddress(""); !reflect.DeepEqual(got, "https://example.com") { t.Errorf("Wrong full path, got: %v", got) } + + assert.Equal(t, "https://example.net", cfg1.getFullAddress("https://example.net")) + assert.Equal(t, "https://example.net", cfg2.getFullAddress("https://example.net")) } func Test_getRelativeBlogPath(t *testing.T) {