mirror of https://github.com/jlelse/GoBlog
Merge branch 'jlelse:master' into master
This commit is contained in:
commit
c45f9fb7ed
27
blogroll.go
27
blogroll.go
|
@ -2,13 +2,14 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"fmt"
|
"context"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/carlmjohnson/requests"
|
||||||
"github.com/kaorimatz/go-opml"
|
"github.com/kaorimatz/go-opml"
|
||||||
"github.com/thoas/go-funk"
|
"github.com/thoas/go-funk"
|
||||||
"go.goblog.app/app/pkgs/contenttype"
|
"go.goblog.app/app/pkgs/contenttype"
|
||||||
|
@ -64,22 +65,16 @@ func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) {
|
||||||
if cache := a.db.loadOutlineCache(blog); cache != nil {
|
if cache := a.db.loadOutlineCache(blog); cache != nil {
|
||||||
return cache, nil
|
return cache, nil
|
||||||
}
|
}
|
||||||
req, err := http.NewRequest(http.MethodGet, config.Opml, nil)
|
rb := requests.URL(config.Opml).Client(a.httpClient).UserAgent(appUserAgent)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if config.AuthHeader != "" && config.AuthValue != "" {
|
if config.AuthHeader != "" && config.AuthValue != "" {
|
||||||
req.Header.Set(config.AuthHeader, config.AuthValue)
|
rb.Header(config.AuthHeader, config.AuthValue)
|
||||||
}
|
}
|
||||||
res, err := a.httpClient.Do(req)
|
var o *opml.OPML
|
||||||
if err != nil {
|
err := rb.Handle(func(r *http.Response) (err error) {
|
||||||
return nil, err
|
defer r.Body.Close()
|
||||||
}
|
o, err = opml.Parse(r.Body)
|
||||||
defer res.Body.Close()
|
return
|
||||||
if code := res.StatusCode; code < 200 || 300 <= code {
|
}).Fetch(context.Background())
|
||||||
return nil, fmt.Errorf("opml request not successful, status code: %d", code)
|
|
||||||
}
|
|
||||||
o, err := opml.Parse(res.Body)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -117,7 +112,7 @@ func (db *database) loadOutlineCache(blog string) []*opml.Outline {
|
||||||
if err != nil || data == nil {
|
if err != nil || data == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
o, err := opml.NewParser(bytes.NewReader(data)).Parse()
|
o, err := opml.Parse(bytes.NewReader(data))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,9 +1,11 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"net/http"
|
"net/http"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/carlmjohnson/requests"
|
||||||
"github.com/mmcdole/gofeed"
|
"github.com/mmcdole/gofeed"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
@ -17,6 +19,7 @@ func Test_feeds(t *testing.T) {
|
||||||
_ = app.initDatabase(false)
|
_ = app.initDatabase(false)
|
||||||
app.initComponents(false)
|
app.initComponents(false)
|
||||||
app.d, _ = app.buildRouter()
|
app.d, _ = app.buildRouter()
|
||||||
|
handlerClient := newHandlerClient(app.d)
|
||||||
|
|
||||||
err := app.createPost(&post{
|
err := app.createPost(&post{
|
||||||
Path: "/testpost",
|
Path: "/testpost",
|
||||||
|
@ -26,21 +29,18 @@ func Test_feeds(t *testing.T) {
|
||||||
Parameters: map[string][]string{"title": {"Test Post"}},
|
Parameters: map[string][]string{"title": {"Test Post"}},
|
||||||
Content: "Test Content",
|
Content: "Test Content",
|
||||||
})
|
})
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
for _, typ := range []feedType{rssFeed, atomFeed, jsonFeed} {
|
for _, typ := range []feedType{rssFeed, atomFeed, jsonFeed} {
|
||||||
req, _ := http.NewRequest(http.MethodGet, "http://localhost:8080/posts."+string(typ), nil)
|
var feed *gofeed.Feed
|
||||||
res, err := doHandlerRequest(req, app.d)
|
err := requests.URL("http://localhost:8080/posts." + string(typ)).Client(handlerClient).
|
||||||
|
Handle(func(r *http.Response) (err error) {
|
||||||
require.NoError(t, err)
|
fp := gofeed.NewParser()
|
||||||
|
defer r.Body.Close()
|
||||||
require.Equal(t, http.StatusOK, res.StatusCode)
|
feed, err = fp.Parse(r.Body)
|
||||||
|
return
|
||||||
fp := gofeed.NewParser()
|
}).
|
||||||
feed, err := fp.Parse(res.Body)
|
Fetch(context.Background())
|
||||||
_ = res.Body.Close()
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NotNil(t, feed)
|
require.NotNil(t, feed)
|
||||||
|
|
||||||
|
|
36
geo.go
36
geo.go
|
@ -1,14 +1,14 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
"embed"
|
"embed"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
gogeouri "git.jlel.se/jlelse/go-geouri"
|
gogeouri "git.jlel.se/jlelse/go-geouri"
|
||||||
|
"github.com/carlmjohnson/requests"
|
||||||
geojson "github.com/paulmach/go.geojson"
|
geojson "github.com/paulmach/go.geojson"
|
||||||
"github.com/thoas/go-funk"
|
"github.com/thoas/go-funk"
|
||||||
)
|
)
|
||||||
|
@ -39,33 +39,19 @@ func (a *goBlog) photonReverse(lat, lon float64, lang string) ([]byte, error) {
|
||||||
if cache != nil {
|
if cache != nil {
|
||||||
return cache, nil
|
return cache, nil
|
||||||
}
|
}
|
||||||
uv := url.Values{}
|
var buf bytes.Buffer
|
||||||
uv.Set("lat", fmt.Sprintf("%v", lat))
|
rb := requests.URL("https://photon.komoot.io/reverse").Client(a.httpClient).UserAgent(appUserAgent).ToBytesBuffer(&buf)
|
||||||
uv.Set("lon", fmt.Sprintf("%v", lon))
|
rb.Param("lat", fmt.Sprintf("%v", lat)).Param("lon", fmt.Sprintf("%v", lon))
|
||||||
if lang == "de" || lang == "fr" || lang == "it" {
|
if lang == "de" || lang == "fr" || lang == "it" {
|
||||||
uv.Set("lang", lang)
|
rb.Param("lang", lang)
|
||||||
} else {
|
} else {
|
||||||
uv.Set("lang", "en")
|
rb.Param("lang", "en")
|
||||||
}
|
}
|
||||||
req, err := http.NewRequest(http.MethodGet, "https://photon.komoot.io/reverse?"+uv.Encode(), nil)
|
if err := rb.Fetch(context.Background()); err != nil {
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
req.Header.Set(userAgent, appUserAgent)
|
_ = a.db.cachePersistently(cacheKey, buf.Bytes())
|
||||||
resp, err := a.httpClient.Do(req)
|
return buf.Bytes(), nil
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return nil, fmt.Errorf("response status code: %v", resp.StatusCode)
|
|
||||||
}
|
|
||||||
ba, err := io.ReadAll(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
_ = a.db.cachePersistently(cacheKey, ba)
|
|
||||||
return ba, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func geoOSMLink(g *gogeouri.Geo) string {
|
func geoOSMLink(g *gogeouri.Geo) string {
|
||||||
|
|
2
go.mod
2
go.mod
|
@ -11,7 +11,7 @@ require (
|
||||||
github.com/alecthomas/chroma v0.9.4
|
github.com/alecthomas/chroma v0.9.4
|
||||||
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
|
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
|
||||||
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
|
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
|
||||||
github.com/carlmjohnson/requests v0.21.13
|
github.com/carlmjohnson/requests v0.22.1
|
||||||
github.com/cretz/bine v0.2.0
|
github.com/cretz/bine v0.2.0
|
||||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
|
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
|
||||||
github.com/dgraph-io/ristretto v0.1.0
|
github.com/dgraph-io/ristretto v0.1.0
|
||||||
|
|
4
go.sum
4
go.sum
|
@ -65,8 +65,8 @@ github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8
|
||||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||||
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 h1:t8KYCwSKsOEZBFELI4Pn/phbp38iJ1RRAkDFNin1aak=
|
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 h1:t8KYCwSKsOEZBFELI4Pn/phbp38iJ1RRAkDFNin1aak=
|
||||||
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
|
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
|
||||||
github.com/carlmjohnson/requests v0.21.13 h1:p9DiBwbrLG8uA67YPOrfGMG1ZRzRyPBaO9hXQpX+Ork=
|
github.com/carlmjohnson/requests v0.22.1 h1:YoifpEbpJW4LPRX/+0dJe3vTLducEE9Ib10k6lElIUM=
|
||||||
github.com/carlmjohnson/requests v0.21.13/go.mod h1:Hw4fFOk3xDlHQbNRTGo4oc52TUTpVEq93sNy/H+mrQM=
|
github.com/carlmjohnson/requests v0.22.1/go.mod h1:Hw4fFOk3xDlHQbNRTGo4oc52TUTpVEq93sNy/H+mrQM=
|
||||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
|
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
|
||||||
|
|
|
@ -15,25 +15,21 @@ type fakeHttpClient struct {
|
||||||
|
|
||||||
func newFakeHttpClient() *fakeHttpClient {
|
func newFakeHttpClient() *fakeHttpClient {
|
||||||
fc := &fakeHttpClient{}
|
fc := &fakeHttpClient{}
|
||||||
fc.Client = &http.Client{
|
fc.Client = newHandlerClient(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
||||||
Transport: &handlerRoundTripper{
|
fc.req = r
|
||||||
handler: http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
if fc.handler != nil {
|
||||||
fc.req = r
|
rec := httptest.NewRecorder()
|
||||||
if fc.handler != nil {
|
fc.handler.ServeHTTP(rec, r)
|
||||||
rec := httptest.NewRecorder()
|
fc.res = rec.Result()
|
||||||
fc.handler.ServeHTTP(rec, r)
|
// Copy the headers from the response recorder
|
||||||
fc.res = rec.Result()
|
for k, v := range rec.Header() {
|
||||||
// Copy the headers from the response recorder
|
rw.Header()[k] = v
|
||||||
for k, v := range rec.Header() {
|
}
|
||||||
rw.Header()[k] = v
|
// Copy result status code and body
|
||||||
}
|
rw.WriteHeader(fc.res.StatusCode)
|
||||||
// Copy result status code and body
|
_, _ = io.Copy(rw, rec.Body)
|
||||||
rw.WriteHeader(fc.res.StatusCode)
|
}
|
||||||
_, _ = io.Copy(rw, rec.Body)
|
}))
|
||||||
}
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return fc
|
return fc
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -51,24 +51,14 @@ func Test_indieAuthServer(t *testing.T) {
|
||||||
_ = app.initDatabase(false)
|
_ = app.initDatabase(false)
|
||||||
app.initComponents(false)
|
app.initComponents(false)
|
||||||
|
|
||||||
app.ias.Client = &http.Client{
|
app.ias.Client = newHandlerClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
Transport: &handlerRoundTripper{
|
w.WriteHeader(http.StatusOK)
|
||||||
handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
}))
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
iac := indieauth.NewClient(
|
iac := indieauth.NewClient(
|
||||||
"https://example.com/",
|
"https://example.com/",
|
||||||
"https://example.com/redirect",
|
"https://example.com/redirect",
|
||||||
&http.Client{
|
newHandlerClient(app.d),
|
||||||
Transport: &handlerRoundTripper{
|
|
||||||
handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
app.d.ServeHTTP(w, r)
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
require.NotNil(t, iac)
|
require.NotNil(t, iac)
|
||||||
|
|
||||||
|
|
|
@ -63,7 +63,7 @@ func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc *http
|
||||||
err := requests.
|
err := requests.
|
||||||
URL("https://api.shortpixel.com/v2/reducer-sync.php").
|
URL("https://api.shortpixel.com/v2/reducer-sync.php").
|
||||||
Client(hc).
|
Client(hc).
|
||||||
Post().
|
Method(http.MethodPost).
|
||||||
BodyJSON(map[string]interface{}{
|
BodyJSON(map[string]interface{}{
|
||||||
"key": sp.key,
|
"key": sp.key,
|
||||||
"plugin_version": "GB001",
|
"plugin_version": "GB001",
|
||||||
|
@ -90,41 +90,41 @@ type tinify struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (string, error) {
|
func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Client) (string, error) {
|
||||||
|
tinifyErr := errors.New("failed to compress image using tinify")
|
||||||
// Check url
|
// Check url
|
||||||
fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png")
|
fileExtension, allowed := urlHasExt(url, "jpg", "jpeg", "png")
|
||||||
if !allowed {
|
if !allowed {
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
// Compress
|
// Compress
|
||||||
compressedLocation := ""
|
headers := http.Header{}
|
||||||
err := requests.
|
err := requests.
|
||||||
URL("https://api.tinify.com/shrink").
|
URL("https://api.tinify.com/shrink").
|
||||||
Client(hc).
|
Client(hc).
|
||||||
Post().
|
Method(http.MethodPost).
|
||||||
BasicAuth("api", tf.key).
|
BasicAuth("api", tf.key).
|
||||||
BodyJSON(map[string]interface{}{
|
BodyJSON(map[string]interface{}{
|
||||||
"source": map[string]interface{}{
|
"source": map[string]interface{}{
|
||||||
"url": url,
|
"url": url,
|
||||||
},
|
},
|
||||||
}).
|
}).
|
||||||
Handle(func(r *http.Response) error {
|
ToHeaders(headers).
|
||||||
compressedLocation = r.Header.Get("Location")
|
|
||||||
if compressedLocation == "" {
|
|
||||||
return errors.New("location header missing")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}).
|
|
||||||
Fetch(context.Background())
|
Fetch(context.Background())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println("Tinify error:", err.Error())
|
log.Println("Tinify error:", err.Error())
|
||||||
return "", errors.New("failed to compress image using tinify")
|
return "", tinifyErr
|
||||||
|
}
|
||||||
|
compressedLocation := headers.Get("Location")
|
||||||
|
if compressedLocation == "" {
|
||||||
|
log.Println("Tinify error: location header missing")
|
||||||
|
return "", tinifyErr
|
||||||
}
|
}
|
||||||
// Resize and download image
|
// Resize and download image
|
||||||
var imgBuffer bytes.Buffer
|
var imgBuffer bytes.Buffer
|
||||||
err = requests.
|
err = requests.
|
||||||
URL(compressedLocation).
|
URL(compressedLocation).
|
||||||
Client(hc).
|
Client(hc).
|
||||||
Post().
|
Method(http.MethodPost).
|
||||||
BasicAuth("api", tf.key).
|
BasicAuth("api", tf.key).
|
||||||
BodyJSON(map[string]interface{}{
|
BodyJSON(map[string]interface{}{
|
||||||
"resize": map[string]interface{}{
|
"resize": map[string]interface{}{
|
||||||
|
@ -137,7 +137,7 @@ func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Cli
|
||||||
Fetch(context.Background())
|
Fetch(context.Background())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println("Tinify error:", err.Error())
|
log.Println("Tinify error:", err.Error())
|
||||||
return "", errors.New("failed to compress image using tinify")
|
return "", tinifyErr
|
||||||
}
|
}
|
||||||
// Upload compressed file
|
// Upload compressed file
|
||||||
return uploadCompressedFile(fileExtension, &imgBuffer, upload)
|
return uploadCompressedFile(fileExtension, &imgBuffer, upload)
|
||||||
|
@ -157,7 +157,6 @@ func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http
|
||||||
err := requests.
|
err := requests.
|
||||||
URL(fmt.Sprintf("https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=%d,h=%d/%s", defaultCompressionWidth, defaultCompressionHeight, url)).
|
URL(fmt.Sprintf("https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=%d,h=%d/%s", defaultCompressionWidth, defaultCompressionHeight, url)).
|
||||||
Client(hc).
|
Client(hc).
|
||||||
Get().
|
|
||||||
ToBytesBuffer(&imgBuffer).
|
ToBytesBuffer(&imgBuffer).
|
||||||
Fetch(context.Background())
|
Fetch(context.Background())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
3
ntfy.go
3
ntfy.go
|
@ -2,6 +2,7 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/carlmjohnson/requests"
|
"github.com/carlmjohnson/requests"
|
||||||
|
@ -22,7 +23,7 @@ func (a *goBlog) sendNtfy(cfg *configNtfy, msg string) error {
|
||||||
URL(cfg.Topic).
|
URL(cfg.Topic).
|
||||||
Client(a.httpClient).
|
Client(a.httpClient).
|
||||||
UserAgent(appUserAgent).
|
UserAgent(appUserAgent).
|
||||||
Post().
|
Method(http.MethodPost).
|
||||||
BodyReader(strings.NewReader(msg)).
|
BodyReader(strings.NewReader(msg)).
|
||||||
Fetch(context.Background())
|
Fetch(context.Background())
|
||||||
}
|
}
|
||||||
|
|
|
@ -26,7 +26,9 @@ func (a *goBlog) checkDeletedPosts() {
|
||||||
for _, post := range postsToDelete {
|
for _, post := range postsToDelete {
|
||||||
// Check if post is deleted for more than 7 days
|
// Check if post is deleted for more than 7 days
|
||||||
if deleted, err := dateparse.ParseLocal(post.firstParameter("deleted")); err == nil && deleted.Add(time.Hour*24*7).Before(time.Now()) {
|
if deleted, err := dateparse.ParseLocal(post.firstParameter("deleted")); err == nil && deleted.Add(time.Hour*24*7).Before(time.Now()) {
|
||||||
a.deletePost(post.Path)
|
if err := a.deletePost(post.Path); err != nil {
|
||||||
|
log.Println("Error deleting post:", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,12 +16,13 @@ func Test_checkDeletedPosts(t *testing.T) {
|
||||||
app.initComponents(false)
|
app.initComponents(false)
|
||||||
|
|
||||||
// Create a post
|
// Create a post
|
||||||
app.createPost(&post{
|
err := app.createPost(&post{
|
||||||
Content: "Test",
|
Content: "Test",
|
||||||
Status: statusPublished,
|
Status: statusPublished,
|
||||||
Path: "/testpost",
|
Path: "/testpost",
|
||||||
Section: "posts",
|
Section: "posts",
|
||||||
})
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Check if post count is 1
|
// Check if post count is 1
|
||||||
count, err := app.db.countPosts(&postsRequestConfig{})
|
count, err := app.db.countPosts(&postsRequestConfig{})
|
||||||
|
@ -49,7 +50,8 @@ func Test_checkDeletedPosts(t *testing.T) {
|
||||||
require.Equal(t, 1, count)
|
require.Equal(t, 1, count)
|
||||||
|
|
||||||
// Set deleted time to more than 7 days ago
|
// Set deleted time to more than 7 days ago
|
||||||
app.db.replacePostParam("/testpost", "deleted", []string{time.Now().Add(-time.Hour * 24 * 8).Format(time.RFC3339)})
|
err = app.db.replacePostParam("/testpost", "deleted", []string{time.Now().Add(-time.Hour * 24 * 8).Format(time.RFC3339)})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Run deleter
|
// Run deleter
|
||||||
app.checkDeletedPosts()
|
app.checkDeletedPosts()
|
||||||
|
|
|
@ -8,7 +8,6 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
gogeouri "git.jlel.se/jlelse/go-geouri"
|
gogeouri "git.jlel.se/jlelse/go-geouri"
|
||||||
"github.com/PuerkitoBio/goquery"
|
|
||||||
"github.com/araddon/dateparse"
|
"github.com/araddon/dateparse"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
@ -113,11 +112,9 @@ func (a *goBlog) postSummary(p *post) (summary string) {
|
||||||
}
|
}
|
||||||
html := string(a.postHtml(p, false))
|
html := string(a.postHtml(p, false))
|
||||||
if splitted := strings.Split(html, summaryDivider); len(splitted) > 1 {
|
if splitted := strings.Split(html, summaryDivider); len(splitted) > 1 {
|
||||||
doc, _ := goquery.NewDocumentFromReader(strings.NewReader(splitted[0]))
|
summary = htmlText(splitted[0])
|
||||||
summary = doc.Text()
|
|
||||||
} else {
|
} else {
|
||||||
doc, _ := goquery.NewDocumentFromReader(strings.NewReader(html))
|
summary = strings.Split(htmlText(html), "\n\n")[0]
|
||||||
summary = doc.Find("p").First().Text()
|
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,11 +34,7 @@ func Test_serveDate(t *testing.T) {
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
client := &http.Client{
|
client := newHandlerClient(app.d)
|
||||||
Transport: &handlerRoundTripper{
|
|
||||||
handler: app.d,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
var resString string
|
var resString string
|
||||||
|
|
||||||
|
@ -128,11 +124,7 @@ func Test_servePost(t *testing.T) {
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
client := &http.Client{
|
client := newHandlerClient(app.d)
|
||||||
Transport: &handlerRoundTripper{
|
|
||||||
handler: app.d,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
var resString string
|
var resString string
|
||||||
|
|
||||||
|
|
|
@ -36,11 +36,7 @@ func Test_sitemap(t *testing.T) {
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
client := &http.Client{
|
client := newHandlerClient(app.d)
|
||||||
Transport: &handlerRoundTripper{
|
|
||||||
handler: app.d,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
var resString string
|
var resString string
|
||||||
|
|
||||||
|
|
8
tts.go
8
tts.go
|
@ -8,6 +8,7 @@ import (
|
||||||
"html"
|
"html"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"path"
|
"path"
|
||||||
"strings"
|
"strings"
|
||||||
|
@ -63,7 +64,10 @@ func (a *goBlog) createPostTTSAudio(p *post) error {
|
||||||
ssml.WriteString("<speak>")
|
ssml.WriteString("<speak>")
|
||||||
ssml.WriteString(html.EscapeString(a.renderMdTitle(p.Title())))
|
ssml.WriteString(html.EscapeString(a.renderMdTitle(p.Title())))
|
||||||
ssml.WriteString("<break time=\"1s\"/>")
|
ssml.WriteString("<break time=\"1s\"/>")
|
||||||
ssml.WriteString(html.EscapeString(cleanHTMLText(string(a.postHtml(p, false)))))
|
for _, part := range strings.Split(htmlText(string(a.postHtml(p, false))), "\n\n") {
|
||||||
|
ssml.WriteString(html.EscapeString(part))
|
||||||
|
ssml.WriteString("<break time=\"500ms\"/>")
|
||||||
|
}
|
||||||
ssml.WriteString("</speak>")
|
ssml.WriteString("</speak>")
|
||||||
|
|
||||||
// Generate audio
|
// Generate audio
|
||||||
|
@ -177,7 +181,7 @@ func (a *goBlog) createTTSAudio(lang, ssml string, w io.Writer) error {
|
||||||
Param("key", gctts.GoogleAPIKey).
|
Param("key", gctts.GoogleAPIKey).
|
||||||
Client(a.httpClient).
|
Client(a.httpClient).
|
||||||
UserAgent(appUserAgent).
|
UserAgent(appUserAgent).
|
||||||
Post().
|
Method(http.MethodPost).
|
||||||
BodyJSON(body).
|
BodyJSON(body).
|
||||||
ToJSON(&response).
|
ToJSON(&response).
|
||||||
Fetch(context.Background())
|
Fetch(context.Background())
|
||||||
|
|
57
utils.go
57
utils.go
|
@ -246,24 +246,48 @@ func mBytesString(size int64) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func htmlText(s string) string {
|
func htmlText(s string) string {
|
||||||
doc, _ := goquery.NewDocumentFromReader(strings.NewReader(s))
|
// Build policy to only allow a subset of HTML tags
|
||||||
|
textPolicy := bluemonday.StrictPolicy()
|
||||||
|
textPolicy.AllowElements("h1", "h2", "h3", "h4", "h5", "h6") // Headers
|
||||||
|
textPolicy.AllowElements("p") // Paragraphs
|
||||||
|
textPolicy.AllowElements("ol", "ul", "li") // Lists
|
||||||
|
textPolicy.AllowElements("blockquote") // Blockquotes
|
||||||
|
// Filter HTML tags
|
||||||
|
htmlBuf := textPolicy.SanitizeReader(strings.NewReader(s))
|
||||||
|
// Read HTML into document
|
||||||
|
doc, _ := goquery.NewDocumentFromReader(htmlBuf)
|
||||||
var text strings.Builder
|
var text strings.Builder
|
||||||
paragraphs := doc.Find("p")
|
if bodyChild := doc.Find("body").Children(); bodyChild.Length() > 0 {
|
||||||
if paragraphs.Length() == 0 {
|
// Input was real HTML, so build the text from the body
|
||||||
text.WriteString(doc.Text())
|
// Declare recursive function to print childs
|
||||||
|
var printChilds func(childs *goquery.Selection)
|
||||||
|
printChilds = func(childs *goquery.Selection) {
|
||||||
|
childs.Each(func(i int, sel *goquery.Selection) {
|
||||||
|
if i > 0 && // Not first child
|
||||||
|
sel.Is("h1, h2, h3, h4, h5, h6, p, ol, ul, li, blockquote") { // All elements that start a new paragraph
|
||||||
|
text.WriteString("\n\n")
|
||||||
|
}
|
||||||
|
if sel.Is("ol > li") { // List item in ordered list
|
||||||
|
fmt.Fprintf(&text, "%d. ", i+1) // Add list item number
|
||||||
|
}
|
||||||
|
if sel.Children().Length() > 0 { // Has children
|
||||||
|
printChilds(sel.Children()) // Recursive call to print childs
|
||||||
|
} else {
|
||||||
|
text.WriteString(sel.Text()) // Print text
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
printChilds(bodyChild)
|
||||||
} else {
|
} else {
|
||||||
paragraphs.Each(func(i int, s *goquery.Selection) {
|
// Input was probably just text, so just use the text
|
||||||
if i > 0 {
|
text.WriteString(doc.Text())
|
||||||
text.WriteString("\n\n")
|
|
||||||
}
|
|
||||||
text.WriteString(s.Text())
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
r := strings.TrimSpace(text.String())
|
// Trim whitespace and return
|
||||||
return r
|
return strings.TrimSpace(text.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
func cleanHTMLText(s string) string {
|
func cleanHTMLText(s string) string {
|
||||||
|
// Clean HTML with UGC policy and return text
|
||||||
return htmlText(bluemonday.UGCPolicy().Sanitize(s))
|
return htmlText(bluemonday.UGCPolicy().Sanitize(s))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -302,14 +326,15 @@ func (rt *handlerRoundTripper) RoundTrip(req *http.Request) (*http.Response, err
|
||||||
return nil, errors.New("no handler")
|
return nil, errors.New("no handler")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newHandlerClient(handler http.Handler) *http.Client {
|
||||||
|
return &http.Client{Transport: &handlerRoundTripper{handler: handler}}
|
||||||
|
}
|
||||||
|
|
||||||
func doHandlerRequest(req *http.Request, handler http.Handler) (*http.Response, error) {
|
func doHandlerRequest(req *http.Request, handler http.Handler) (*http.Response, error) {
|
||||||
client := &http.Client{
|
|
||||||
Transport: &handlerRoundTripper{handler: handler},
|
|
||||||
}
|
|
||||||
if req.URL.Path == "" {
|
if req.URL.Path == "" {
|
||||||
req.URL.Path = "/"
|
req.URL.Path = "/"
|
||||||
}
|
}
|
||||||
return client.Do(req)
|
return newHandlerClient(handler).Do(req)
|
||||||
}
|
}
|
||||||
|
|
||||||
func saveToFile(reader io.Reader, fileName string) error {
|
func saveToFile(reader io.Reader, fileName string) error {
|
||||||
|
|
|
@ -76,13 +76,33 @@ func Test_urlHasExt(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func Test_cleanHTMLText(t *testing.T) {
|
func Test_htmlText(t *testing.T) {
|
||||||
assert.Equal(t, `"This is a 'test'" 😁`, cleanHTMLText(`"This is a 'test'" 😁`))
|
// Text without HTML
|
||||||
assert.Equal(t, `Test`, cleanHTMLText(`<b>Test</b>`))
|
assert.Equal(t, "This is a test", htmlText("This is a test"))
|
||||||
assert.Equal(t, "Test\n\nTest", cleanHTMLText(`<p>Test</p><p>Test</p>`))
|
// Text without HTML and Emojis
|
||||||
assert.Equal(t, "Test\n\nTest", cleanHTMLText("<p>Test</p>\n<p>Test</p>"))
|
assert.Equal(t, "This is a test 😁", htmlText("This is a test 😁"))
|
||||||
assert.Equal(t, "Test\n\nTest", cleanHTMLText("<div><p>Test</p>\n<p>Test</p></div>"))
|
// Text without HTML and quoutes
|
||||||
assert.Equal(t, "Test test\n\nTest", cleanHTMLText(`<p>Test <b>test</b></p><p>Test</p>`))
|
assert.Equal(t, "This is a 'test'", htmlText("This is a 'test'"))
|
||||||
|
// Text with formatting (like bold or italic)
|
||||||
|
assert.Equal(t, "This is a test", htmlText("<b>This is a test</b>"))
|
||||||
|
assert.Equal(t, "This is a test", htmlText("<i>This is a test</i>"))
|
||||||
|
// Unordered list
|
||||||
|
assert.Equal(t, "Test\n\nTest", htmlText(`<ul><li>Test</li><li>Test</li></ul>`))
|
||||||
|
// Ordered list
|
||||||
|
assert.Equal(t, "1. Test\n\n2. Test", htmlText(`<ol><li>Test</li><li>Test</li></ol>`))
|
||||||
|
// Nested unordered list
|
||||||
|
assert.Equal(t, "Test\n\nTest\n\nTest", htmlText(`<ul><li>Test</li><li><ul><li>Test</li><li>Test</li></ul></li></ul>`))
|
||||||
|
// Headers and paragraphs
|
||||||
|
assert.Equal(t, "Test\n\nTest", htmlText(`<h1>Test</h1><p>Test</p>`))
|
||||||
|
assert.Equal(t, "Test\n\nTest\n\nTest", htmlText(`<h1>Test</h1><p>Test</p><h2>Test</h2>`))
|
||||||
|
// Blockquote
|
||||||
|
assert.Equal(t, "Test\n\nBlockqoute content", htmlText(`<p>Test</p><blockquote><p>Blockqoute content</p></blockquote>`))
|
||||||
|
// Nested blockquotes
|
||||||
|
assert.Equal(t, "Blockqoute content\n\nBlockqoute content", htmlText(`<blockquote><p>Blockqoute content</p><blockquote><p>Blockqoute content</p></blockquote></blockquote>`))
|
||||||
|
// Code (should be ignored)
|
||||||
|
assert.Equal(t, "Test", htmlText(`<p>Test</p><pre><code>Code content</code></pre>`))
|
||||||
|
// Inline code (should not be ignored)
|
||||||
|
assert.Equal(t, "Test Code content", htmlText(`<p>Test <code>Code content</code></p>`))
|
||||||
}
|
}
|
||||||
|
|
||||||
func Test_containsStrings(t *testing.T) {
|
func Test_containsStrings(t *testing.T) {
|
||||||
|
|
|
@ -73,7 +73,7 @@ func (a *goBlog) sendWebmentions(p *post) error {
|
||||||
|
|
||||||
func (a *goBlog) sendWebmention(endpoint, source, target string) error {
|
func (a *goBlog) sendWebmention(endpoint, source, target string) error {
|
||||||
// TODO: Pass all tests from https://webmention.rocks/
|
// TODO: Pass all tests from https://webmention.rocks/
|
||||||
return requests.URL(endpoint).Client(a.httpClient).Post().UserAgent(appUserAgent).
|
return requests.URL(endpoint).Client(a.httpClient).Method(http.MethodPost).UserAgent(appUserAgent).
|
||||||
BodyForm(url.Values{
|
BodyForm(url.Values{
|
||||||
"source": []string{source},
|
"source": []string{source},
|
||||||
"target": []string{target},
|
"target": []string{target},
|
||||||
|
|
Loading…
Reference in New Issue