Fix 405 on HEAD requests, improve webmention verification

This commit is contained in:
Jan-Lukas Else 2021-11-19 17:36:03 +01:00
parent 395123bad9
commit 974892e3e5
12 changed files with 199 additions and 80 deletions

1
go.mod
View File

@ -104,7 +104,6 @@ require (
github.com/spf13/afero v1.6.0 // indirect github.com/spf13/afero v1.6.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.3.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect
github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3 // indirect github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3 // indirect
github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 // indirect github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 // indirect

2
go.sum
View File

@ -462,8 +462,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk= github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk=
github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

15
http.go
View File

@ -40,7 +40,7 @@ func (a *goBlog) startServer() (err error) {
if a.cfg.Server.Logging { if a.cfg.Server.Logging {
h = h.Append(a.logMiddleware) h = h.Append(a.logMiddleware)
} }
h = h.Append(middleware.Recoverer, middleware.Compress(flate.DefaultCompression), middleware.Heartbeat("/ping")) h = h.Append(middleware.Recoverer, middleware.Compress(flate.DefaultCompression), middleware.Heartbeat("/ping"), headAsGetHandler)
if a.httpsConfigured(false) { if a.httpsConfigured(false) {
h = h.Append(a.securityHeaders) h = h.Append(a.securityHeaders)
} }
@ -125,7 +125,6 @@ func (a *goBlog) buildRouter() (http.Handler, error) {
mr.Use(middleware.RedirectSlashes) mr.Use(middleware.RedirectSlashes)
mr.Use(middleware.CleanPath) mr.Use(middleware.CleanPath)
mr.Use(middleware.GetHead)
mr.Group(a.mediaFilesRouter) mr.Group(a.mediaFilesRouter)
@ -139,7 +138,6 @@ func (a *goBlog) buildRouter() (http.Handler, error) {
r.Use(fixHTTPHandler) r.Use(fixHTTPHandler)
r.Use(middleware.RedirectSlashes) r.Use(middleware.RedirectSlashes)
r.Use(middleware.CleanPath) r.Use(middleware.CleanPath)
r.Use(middleware.GetHead)
// Tor // Tor
if a.cfg.Server.Tor { if a.cfg.Server.Tor {
@ -284,3 +282,14 @@ func (a *goBlog) servePostsAliasesRedirects() http.HandlerFunc {
alice.New(a.cacheMiddleware, a.checkRegexRedirects).ThenFunc(a.serve404).ServeHTTP(w, r) alice.New(a.cacheMiddleware, a.checkRegexRedirects).ThenFunc(a.serve404).ServeHTTP(w, r)
} }
} }
func (a *goBlog) getAppRouter() http.Handler {
for {
// Wait until router is ready
if a.d != nil {
break
}
time.Sleep(time.Millisecond * 100)
}
return a.d
}

View File

@ -20,6 +20,15 @@ func fixHTTPHandler(next http.Handler) http.Handler {
}) })
} }
func headAsGetHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodHead {
r.Method = http.MethodGet
}
next.ServeHTTP(w, r)
})
}
func (a *goBlog) securityHeaders(next http.Handler) http.Handler { func (a *goBlog) securityHeaders(next http.Handler) http.Handler {
// Build CSP domains list // Build CSP domains list
var cspBuilder strings.Builder var cspBuilder strings.Builder

View File

@ -19,8 +19,7 @@ By <a class="u-uid u-url url" href="https://example.net/" rel="me"><img class="u
<div class="entry-meta">On <a class="u-url" href="https://example.net/articles/micropub-crossposting-to-twitter-and-enabling-tweetstorms" rel="bookmark"><time datetime="2021-02-22T19:17:05+01:00" class="dt-published">Feb 22, 2021</time></a></div> <div class="entry-meta">On <a class="u-url" href="https://example.net/articles/micropub-crossposting-to-twitter-and-enabling-tweetstorms" rel="bookmark"><time datetime="2021-02-22T19:17:05+01:00" class="dt-published">Feb 22, 2021</time></a></div>
</header> </header>
<div class="e-content"> <div class="e-content">
<p>I&#8217;ve previously talked about how I <a href="https://example.org/articles/micropub-syndication-targets-and-crossposting-to-mastodon">crosspost from this blog to my Mastodon account</a> without the need for a third-party service, and how I leverage WordPress&#8217;s hook system to even enable toot threading.</p> <p>I&#8217;ve previously talked about how I <a href="https://example.org/articles/micropub-syndication-targets-and-crossposting-to-mastodon">crosspost from this blog to my Mastodon account</a> without the need for a third-party service, and how I leverage WordPress&#8217;s hook system to even enable toot threading.</p><p>In this post, I&#8217;m going to really quickly explain my (extremely similar) Twitter setup. (Note: I don&#8217;t actually syndicate <em>this</em> blog&#8217;s posts to Twitter, but I <em>do</em> use this very setup on another site of mine.)</p>
<p>In this post, I&#8217;m going to really quickly explain my (extremely similar) Twitter setup. (Note: I don&#8217;t actually syndicate <em>this</em> blog&#8217;s posts to Twitter, but I <em>do</em> use this very setup on another site of mine.)</p>
<p>I liked the idea of a dead-simple Twitter plugin, so I forked my Mastodon plugin and tweaked a few things here and there. Once I&#8217;ve installed it, and created a developer account, generated the necessary keys, and let WordPress know about them, things look, well, <em>very</em> familiar. In fact, crossposting should now <em>just work</em>.</p> <p>I liked the idea of a dead-simple Twitter plugin, so I forked my Mastodon plugin and tweaked a few things here and there. Once I&#8217;ve installed it, and created a developer account, generated the necessary keys, and let WordPress know about them, things look, well, <em>very</em> familiar. In fact, crossposting should now <em>just work</em>.</p>
<p>Now, to enable this when posting through Micropub rather than WordPress&#8217;s admin interface! Again, since posting through Micropub means no WordPress interface, and thus no &#8220;meta box&#8221; and no checkbox, and no way for WordPress to know if I wanted to crosspost a certain article or not, I&#8217;m going to have to use &#8230; <em>syndication targets</em> (which were invented precisely for this reason).</p> <p>Now, to enable this when posting through Micropub rather than WordPress&#8217;s admin interface! Again, since posting through Micropub means no WordPress interface, and thus no &#8220;meta box&#8221; and no checkbox, and no way for WordPress to know if I wanted to crosspost a certain article or not, I&#8217;m going to have to use &#8230; <em>syndication targets</em> (which were invented precisely for this reason).</p>
</div> </div>

View File

@ -2,9 +2,12 @@ package main
import ( import (
"crypto/sha256" "crypto/sha256"
"errors"
"fmt" "fmt"
"html/template" "html/template"
"io" "io"
"net/http"
"net/http/httptest"
"net/url" "net/url"
"path" "path"
"sort" "sort"
@ -240,15 +243,26 @@ func mBytesString(size int64) string {
} }
func htmlText(s string) string { func htmlText(s string) string {
d, err := goquery.NewDocumentFromReader(strings.NewReader(s)) doc, _ := goquery.NewDocumentFromReader(strings.NewReader(s))
if err != nil { var text strings.Builder
return "" paragraphs := doc.Find("p")
if paragraphs.Length() == 0 {
text.WriteString(doc.Text())
} else {
paragraphs.Each(func(i int, s *goquery.Selection) {
if i > 0 {
text.WriteString("\n\n")
}
text.WriteString(s.Text())
})
} }
return strings.TrimSpace(d.Text()) r := strings.TrimSpace(text.String())
return r
} }
func cleanHTMLText(s string) string { func cleanHTMLText(s string) string {
return htmlText(bluemonday.StrictPolicy().Sanitize(s)) s = bluemonday.UGCPolicy().Sanitize(s)
return htmlText(s)
} }
func defaultIfEmpty(s, d string) string { func defaultIfEmpty(s, d string) string {
@ -267,3 +281,28 @@ func containsStrings(s string, subStrings ...string) bool {
func timeNoErr(t time.Time, _ error) time.Time { func timeNoErr(t time.Time, _ error) time.Time {
return t return t
} }
type handlerRoundTripper struct {
http.RoundTripper
handler http.Handler
}
func (rt *handlerRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
if rt.handler != nil {
// Fake request with handler
rec := httptest.NewRecorder()
rt.handler.ServeHTTP(rec, req)
resp := rec.Result()
// Copy request to response
resp.Request = req
return resp, nil
}
return nil, errors.New("no handler")
}
func doHandlerRequest(req *http.Request, handler http.Handler) (*http.Response, error) {
client := &http.Client{
Transport: &handlerRoundTripper{handler: handler},
}
return client.Do(req)
}

View File

@ -79,6 +79,10 @@ func Test_urlHasExt(t *testing.T) {
func Test_cleanHTMLText(t *testing.T) { func Test_cleanHTMLText(t *testing.T) {
assert.Equal(t, `"This is a 'test'" 😁`, cleanHTMLText(`"This is a 'test'" 😁`)) assert.Equal(t, `"This is a 'test'" 😁`, cleanHTMLText(`"This is a 'test'" 😁`))
assert.Equal(t, `Test`, cleanHTMLText(`<b>Test</b>`)) assert.Equal(t, `Test`, cleanHTMLText(`<b>Test</b>`))
assert.Equal(t, "Test\n\nTest", cleanHTMLText(`<p>Test</p><p>Test</p>`))
assert.Equal(t, "Test\n\nTest", cleanHTMLText("<p>Test</p>\n<p>Test</p>"))
assert.Equal(t, "Test\n\nTest", cleanHTMLText("<div><p>Test</p>\n<p>Test</p></div>"))
assert.Equal(t, "Test test\n\nTest", cleanHTMLText(`<p>Test <b>test</b></p><p>Test</p>`))
} }
func Test_containsStrings(t *testing.T) { func Test_containsStrings(t *testing.T) {

View File

@ -25,6 +25,7 @@ type mention struct {
Source string Source string
NewSource string NewSource string
Target string Target string
NewTarget string
Created int64 Created int64
Title string Title string
Content string Content string
@ -52,7 +53,9 @@ func (a *goBlog) handleWebmention(w http.ResponseWriter, r *http.Request) {
a.serveError(w, r, err.Error(), http.StatusBadRequest) a.serveError(w, r, err.Error(), http.StatusBadRequest)
return return
} }
if !strings.HasPrefix(m.Target, a.cfg.Server.PublicAddress) { hasShortPrefix := a.cfg.Server.ShortPublicAddress != "" && strings.HasPrefix(m.Target, a.cfg.Server.ShortPublicAddress)
hasLongPrefix := strings.HasPrefix(m.Target, a.cfg.Server.PublicAddress)
if !hasShortPrefix && !hasLongPrefix {
a.debug("Webmention target not allowed:", m.Target) a.debug("Webmention target not allowed:", m.Target)
a.serveError(w, r, "target not allowed", http.StatusBadRequest) a.serveError(w, r, "target not allowed", http.StatusBadRequest)
return return
@ -89,9 +92,21 @@ func (a *goBlog) extractMention(r *http.Request) (*mention, error) {
}, nil }, nil
} }
func (db *database) webmentionExists(source, target string) bool { func (db *database) webmentionExists(m *mention) bool {
result := 0 result := 0
row, err := db.queryRow("select exists(select 1 from webmentions where lowerx(source) = lowerx(@source) and lowerx(target) = lowerx(@target))", sql.Named("source", source), sql.Named("target", target)) row, err := db.queryRow(
`
select exists(
select 1
from webmentions
where
lowerx(source) in (lowerx(@source), lowerx(@newsource))
and lowerx(target) in (lowerx(@target), lowerx(@newtarget))
)
`,
sql.Named("source", m.Source), sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)),
sql.Named("target", m.Target), sql.Named("newtarget", defaultIfEmpty(m.NewTarget, m.Target)),
)
if err != nil { if err != nil {
return false return false
} }
@ -126,17 +141,56 @@ func (db *database) insertWebmention(m *mention, status webmentionStatus) error
return err return err
} }
func (db *database) deleteWebmention(id int) error { func (db *database) updateWebmention(m *mention, newStatus webmentionStatus) error {
_, err := db.exec(`
update webmentions
set
source = @newsource,
target = @newtarget,
status = @status,
title = @title,
content = @content,
author = @author
where
lowerx(source) in (lowerx(@source), lowerx(@newsource2))
and lowerx(target) in (lowerx(@target), lowerx(@newtarget2))
`,
sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)),
sql.Named("newtarget", defaultIfEmpty(m.NewTarget, m.Target)),
sql.Named("status", newStatus),
sql.Named("title", m.Title),
sql.Named("content", m.Content),
sql.Named("author", m.Author),
sql.Named("source", m.Source),
sql.Named("newsource2", defaultIfEmpty(m.NewSource, m.Source)),
sql.Named("target", m.Target),
sql.Named("newtarget2", defaultIfEmpty(m.NewTarget, m.Target)),
)
return err
}
func (db *database) deleteWebmentionId(id int) error {
_, err := db.exec("delete from webmentions where id = @id", sql.Named("id", id)) _, err := db.exec("delete from webmentions where id = @id", sql.Named("id", id))
return err return err
} }
func (db *database) approveWebmention(id int) error { func (db *database) deleteWebmention(m *mention) error {
_, err := db.exec(
"delete from webmentions where lowerx(source) in (lowerx(@source), lowerx(@newsource)) and lowerx(target) in (lowerx(@target), lowerx(@newtarget))",
sql.Named("source", m.Source),
sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)),
sql.Named("target", m.Target),
sql.Named("newtarget", defaultIfEmpty(m.NewTarget, m.Target)),
)
return err
}
func (db *database) approveWebmentionId(id int) error {
_, err := db.exec("update webmentions set status = ? where id = ?", webmentionStatusApproved, id) _, err := db.exec("update webmentions set status = ? where id = ?", webmentionStatusApproved, id)
return err return err
} }
func (a *goBlog) reverifyWebmention(id int) error { func (a *goBlog) reverifyWebmentionId(id int) error {
m, err := a.db.getWebmentions(&webmentionsRequestConfig{ m, err := a.db.getWebmentions(&webmentionsRequestConfig{
id: id, id: id,
limit: 1, limit: 1,

View File

@ -120,11 +120,11 @@ func (a *goBlog) webmentionAdminAction(w http.ResponseWriter, r *http.Request) {
} }
switch action { switch action {
case "delete": case "delete":
err = a.db.deleteWebmention(id) err = a.db.deleteWebmentionId(id)
case "approve": case "approve":
err = a.db.approveWebmention(id) err = a.db.approveWebmentionId(id)
case "reverify": case "reverify":
err = a.reverifyWebmention(id) err = a.reverifyWebmentionId(id)
} }
if err != nil { if err != nil {
a.serveError(w, r, err.Error(), http.StatusInternalServerError) a.serveError(w, r, err.Error(), http.StatusInternalServerError)

View File

@ -2,20 +2,19 @@ package main
import ( import (
"bytes" "bytes"
"database/sql"
"encoding/gob" "encoding/gob"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"log" "log"
"net/http" "net/http"
"net/http/httptest"
"net/url" "net/url"
"strings" "strings"
"time" "time"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"github.com/thoas/go-funk" "github.com/thoas/go-funk"
"go.goblog.app/app/pkgs/contenttype"
"willnorris.com/go/microformats" "willnorris.com/go/microformats"
) )
@ -62,78 +61,72 @@ func (a *goBlog) queueMention(m *mention) error {
} }
func (a *goBlog) verifyMention(m *mention) error { func (a *goBlog) verifyMention(m *mention) error {
// Do request // Request target
req, err := http.NewRequest(http.MethodGet, m.Source, nil) targetReq, err := http.NewRequest(http.MethodGet, m.Target, nil)
if err != nil { if err != nil {
return err return err
} }
var resp *http.Response targetReq.Header.Set("Accept", contenttype.HTMLUTF8)
if strings.HasPrefix(m.Source, a.cfg.Server.PublicAddress) { setLoggedIn(targetReq, true)
rec := httptest.NewRecorder() targetResp, err := doHandlerRequest(targetReq, a.getAppRouter())
for a.d == nil { if err != nil {
// Server not yet started return err
time.Sleep(1 * time.Second) }
} // Check if target has a valid status code
setLoggedIn(req, true) if targetResp.StatusCode != http.StatusOK {
a.d.ServeHTTP(rec, req) return a.db.deleteWebmention(m)
resp = rec.Result() }
} else { // Check if target has a redirect
req.Header.Set(userAgent, appUserAgent) if respReq := targetResp.Request; respReq != nil {
resp, err = a.httpClient.Do(req) if ru := respReq.URL; m.Target != ru.String() {
if err != nil { m.NewTarget = ru.String()
return err
} }
} }
// Request source
sourceReq, err := http.NewRequest(http.MethodGet, m.Source, nil)
if err != nil {
return err
}
sourceReq.Header.Set("Accept", contenttype.HTMLUTF8)
var sourceResp *http.Response
if strings.HasPrefix(m.Source, a.cfg.Server.PublicAddress) ||
(a.cfg.Server.ShortPublicAddress != "" && strings.HasPrefix(m.Source, a.cfg.Server.ShortPublicAddress)) {
setLoggedIn(sourceReq, true)
sourceResp, err = doHandlerRequest(sourceReq, a.getAppRouter())
} else {
sourceReq.Header.Set(userAgent, appUserAgent)
sourceResp, err = a.httpClient.Do(sourceReq)
}
if err != nil {
return err
}
// Check if source has a valid status code
if sourceResp.StatusCode != http.StatusOK {
return a.db.deleteWebmention(m)
}
// Check if source has a redirect // Check if source has a redirect
if respReq := resp.Request; respReq != nil { if respReq := sourceResp.Request; respReq != nil {
if ru := respReq.URL; m.Source != ru.String() { if ru := respReq.URL; m.Source != ru.String() {
m.NewSource = ru.String() m.NewSource = ru.String()
} }
} }
// Parse response body // Parse response body
err = m.verifyReader(resp.Body) err = m.verifyReader(sourceResp.Body)
_ = resp.Body.Close() _ = sourceResp.Body.Close()
if err != nil { if err != nil {
// Delete webmentions with old or new source return a.db.deleteWebmention(m)
_, err := a.db.exec(
"delete from webmentions where lowerx(source) in (lowerx(@source), lowerx(@newsource)) and lowerx(target) = lowerx(@target)",
sql.Named("source", m.Source),
sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)),
sql.Named("target", m.Target),
)
return err
} }
if cr := []rune(m.Content); len(cr) > 500 { if cr := []rune(m.Content); len(cr) > 500 {
m.Content = string(cr[0:497]) + "…" m.Content = string(cr[0:497]) + "…"
} }
m.Content = strings.ReplaceAll(m.Content, "\n", " ")
if tr := []rune(m.Title); len(tr) > 60 { if tr := []rune(m.Title); len(tr) > 60 {
m.Title = string(tr[0:57]) + "…" m.Title = string(tr[0:57]) + "…"
} }
newStatus := webmentionStatusVerified newStatus := webmentionStatusVerified
if a.db.webmentionExists(m.Source, m.Target) { // Update or insert webmention
// Check if webmention also has webmention with new source if a.db.webmentionExists(m) {
if m.NewSource != "" && a.db.webmentionExists(m.NewSource, m.Target) {
// Delete it
_, err = a.db.exec(
"delete from webmentions where lowerx(source) = lowerx(@source) and lowerx(target) = lowerx(@target)",
sql.Named("source", m.NewSource), sql.Named("target", m.Target),
)
if err != nil {
return err
}
}
// Update webmention // Update webmention
_, err = a.db.exec( err = a.db.updateWebmention(m, newStatus)
"update webmentions set source = @newsource, status = @status, title = @title, content = @content, author = @author where lowerx(source) = lowerx(@source) and lowerx(target) = lowerx(@target)",
sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)),
sql.Named("status", newStatus),
sql.Named("title", m.Title),
sql.Named("content", m.Content),
sql.Named("author", m.Author),
sql.Named("source", m.Source),
sql.Named("target", m.Target),
)
if err != nil { if err != nil {
return err return err
} }
@ -141,11 +134,14 @@ func (a *goBlog) verifyMention(m *mention) error {
if m.NewSource != "" { if m.NewSource != "" {
m.Source = m.NewSource m.Source = m.NewSource
} }
if m.NewTarget != "" {
m.Target = m.NewTarget
}
err = a.db.insertWebmention(m, newStatus) err = a.db.insertWebmention(m, newStatus)
if err != nil { if err != nil {
return err return err
} }
a.sendNotification(fmt.Sprintf("New webmention from %s to %s", m.Source, m.Target)) a.sendNotification(fmt.Sprintf("New webmention from %s to %s", defaultIfEmpty(m.NewSource, m.Source), defaultIfEmpty(m.NewTarget, m.Target)))
} }
return err return err
} }
@ -161,7 +157,7 @@ func (m *mention) verifyReader(body io.Reader) error {
return err return err
} }
if _, hasLink := funk.FindString(links, func(s string) bool { if _, hasLink := funk.FindString(links, func(s string) bool {
return unescapedPath(s) == unescapedPath(m.Target) return unescapedPath(s) == unescapedPath(m.Target) || unescapedPath(s) == unescapedPath(m.NewTarget)
}); !hasLink { }); !hasLink {
return errors.New("target not found in source") return errors.New("target not found in source")
} }
@ -233,6 +229,12 @@ func (m *mention) fillContent(mf *microformats.Microformat) {
if content, ok := contents[0].(map[string]string); ok { if content, ok := contents[0].(map[string]string); ok {
if contentHTML, ok := content["html"]; ok { if contentHTML, ok := content["html"]; ok {
m.Content = cleanHTMLText(contentHTML) m.Content = cleanHTMLText(contentHTML)
// Replace newlines with spaces
m.Content = strings.ReplaceAll(m.Content, "\n", " ")
// Collapse double spaces
m.Content = strings.Join(strings.Fields(m.Content), " ")
// Trim spaces
m.Content = strings.TrimSpace(m.Content)
} }
} }
} }

View File

@ -4,6 +4,7 @@ import (
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -28,6 +29,11 @@ func Test_verifyMention(t *testing.T) {
PublicAddress: "https://example.org", PublicAddress: "https://example.org",
}, },
}, },
d: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, r, r.URL.Path[:len(r.URL.Path)-1], http.StatusFound)
}
}),
} }
_ = app.initDatabase(false) _ = app.initDatabase(false)
@ -35,7 +41,7 @@ func Test_verifyMention(t *testing.T) {
m := &mention{ m := &mention{
Source: "https://example.net/articles/micropub-crossposting-to-twitter-and-enabling-tweetstorms", Source: "https://example.net/articles/micropub-crossposting-to-twitter-and-enabling-tweetstorms",
Target: "https://example.org/articles/micropub-syndication-targets-and-crossposting-to-mastodon", Target: "https://example.org/articles/micropub-syndication-targets-and-crossposting-to-mastodon/",
} }
err = app.verifyMention(m) err = app.verifyMention(m)

View File

@ -52,7 +52,7 @@ func Test_webmentions(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, 1, count) assert.Equal(t, 1, count)
exists := app.db.webmentionExists("Https://Example.net/test", "Https://Example.com/TÄst") exists := app.db.webmentionExists(&mention{Source: "Https://Example.net/test", Target: "Https://Example.com/TÄst"})
assert.True(t, exists) assert.True(t, exists)
mentions = app.db.getWebmentionsByAddress("https://example.com/täst") mentions = app.db.getWebmentionsByAddress("https://example.com/täst")
@ -63,7 +63,7 @@ func Test_webmentions(t *testing.T) {
}) })
require.NoError(t, err) require.NoError(t, err)
if assert.Len(t, mentions, 1) { if assert.Len(t, mentions, 1) {
_ = app.db.approveWebmention(mentions[0].ID) _ = app.db.approveWebmentionId(mentions[0].ID)
} }
mentions = app.db.getWebmentionsByAddress("https://example.com/täst") mentions = app.db.getWebmentionsByAddress("https://example.com/täst")