Browse Source

Fix 405 on HEAD requests, improve webmention verification

master
Jan-Lukas Else 2 weeks ago
parent
commit
974892e3e5
  1. 1
      go.mod
  2. 2
      go.sum
  3. 15
      http.go
  4. 9
      httpMiddlewares.go
  5. 3
      testdata/wmtest.html
  6. 49
      utils.go
  7. 4
      utils_test.go
  8. 66
      webmention.go
  9. 6
      webmentionAdmin.go
  10. 110
      webmentionVerification.go
  11. 8
      webmentionVerification_test.go
  12. 4
      webmention_test.go

1
go.mod

@ -104,7 +104,6 @@ require (
github.com/spf13/afero v1.6.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.3.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3 // indirect
github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 // indirect

2
go.sum

@ -462,8 +462,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk=
github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

15
http.go

@ -40,7 +40,7 @@ func (a *goBlog) startServer() (err error) {
if a.cfg.Server.Logging {
h = h.Append(a.logMiddleware)
}
h = h.Append(middleware.Recoverer, middleware.Compress(flate.DefaultCompression), middleware.Heartbeat("/ping"))
h = h.Append(middleware.Recoverer, middleware.Compress(flate.DefaultCompression), middleware.Heartbeat("/ping"), headAsGetHandler)
if a.httpsConfigured(false) {
h = h.Append(a.securityHeaders)
}
@ -125,7 +125,6 @@ func (a *goBlog) buildRouter() (http.Handler, error) {
mr.Use(middleware.RedirectSlashes)
mr.Use(middleware.CleanPath)
mr.Use(middleware.GetHead)
mr.Group(a.mediaFilesRouter)
@ -139,7 +138,6 @@ func (a *goBlog) buildRouter() (http.Handler, error) {
r.Use(fixHTTPHandler)
r.Use(middleware.RedirectSlashes)
r.Use(middleware.CleanPath)
r.Use(middleware.GetHead)
// Tor
if a.cfg.Server.Tor {
@ -284,3 +282,14 @@ func (a *goBlog) servePostsAliasesRedirects() http.HandlerFunc {
alice.New(a.cacheMiddleware, a.checkRegexRedirects).ThenFunc(a.serve404).ServeHTTP(w, r)
}
}
func (a *goBlog) getAppRouter() http.Handler {
for {
// Wait until router is ready
if a.d != nil {
break
}
time.Sleep(time.Millisecond * 100)
}
return a.d
}

9
httpMiddlewares.go

@ -20,6 +20,15 @@ func fixHTTPHandler(next http.Handler) http.Handler {
})
}
func headAsGetHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodHead {
r.Method = http.MethodGet
}
next.ServeHTTP(w, r)
})
}
func (a *goBlog) securityHeaders(next http.Handler) http.Handler {
// Build CSP domains list
var cspBuilder strings.Builder

3
testdata/wmtest.html

@ -19,8 +19,7 @@ By <a class="u-uid u-url url" href="https://example.net/" rel="me"><img class="u
<div class="entry-meta">On <a class="u-url" href="https://example.net/articles/micropub-crossposting-to-twitter-and-enabling-tweetstorms" rel="bookmark"><time datetime="2021-02-22T19:17:05+01:00" class="dt-published">Feb 22, 2021</time></a></div>
</header>
<div class="e-content">
<p>I&#8217;ve previously talked about how I <a href="https://example.org/articles/micropub-syndication-targets-and-crossposting-to-mastodon">crosspost from this blog to my Mastodon account</a> without the need for a third-party service, and how I leverage WordPress&#8217;s hook system to even enable toot threading.</p>
<p>In this post, I&#8217;m going to really quickly explain my (extremely similar) Twitter setup. (Note: I don&#8217;t actually syndicate <em>this</em> blog&#8217;s posts to Twitter, but I <em>do</em> use this very setup on another site of mine.)</p>
<p>I&#8217;ve previously talked about how I <a href="https://example.org/articles/micropub-syndication-targets-and-crossposting-to-mastodon">crosspost from this blog to my Mastodon account</a> without the need for a third-party service, and how I leverage WordPress&#8217;s hook system to even enable toot threading.</p><p>In this post, I&#8217;m going to really quickly explain my (extremely similar) Twitter setup. (Note: I don&#8217;t actually syndicate <em>this</em> blog&#8217;s posts to Twitter, but I <em>do</em> use this very setup on another site of mine.)</p>
<p>I liked the idea of a dead-simple Twitter plugin, so I forked my Mastodon plugin and tweaked a few things here and there. Once I&#8217;ve installed it, and created a developer account, generated the necessary keys, and let WordPress know about them, things look, well, <em>very</em> familiar. In fact, crossposting should now <em>just work</em>.</p>
<p>Now, to enable this when posting through Micropub rather than WordPress&#8217;s admin interface! Again, since posting through Micropub means no WordPress interface, and thus no &#8220;meta box&#8221; and no checkbox, and no way for WordPress to know if I wanted to crosspost a certain article or not, I&#8217;m going to have to use &#8230; <em>syndication targets</em> (which were invented precisely for this reason).</p>
</div>

49
utils.go

@ -2,9 +2,12 @@ package main
import (
"crypto/sha256"
"errors"
"fmt"
"html/template"
"io"
"net/http"
"net/http/httptest"
"net/url"
"path"
"sort"
@ -240,15 +243,26 @@ func mBytesString(size int64) string {
}
func htmlText(s string) string {
d, err := goquery.NewDocumentFromReader(strings.NewReader(s))
if err != nil {
return ""
doc, _ := goquery.NewDocumentFromReader(strings.NewReader(s))
var text strings.Builder
paragraphs := doc.Find("p")
if paragraphs.Length() == 0 {
text.WriteString(doc.Text())
} else {
paragraphs.Each(func(i int, s *goquery.Selection) {
if i > 0 {
text.WriteString("\n\n")
}
text.WriteString(s.Text())
})
}
return strings.TrimSpace(d.Text())
r := strings.TrimSpace(text.String())
return r
}
func cleanHTMLText(s string) string {
return htmlText(bluemonday.StrictPolicy().Sanitize(s))
s = bluemonday.UGCPolicy().Sanitize(s)
return htmlText(s)
}
func defaultIfEmpty(s, d string) string {
@ -267,3 +281,28 @@ func containsStrings(s string, subStrings ...string) bool {
func timeNoErr(t time.Time, _ error) time.Time {
return t
}
type handlerRoundTripper struct {
http.RoundTripper
handler http.Handler
}
func (rt *handlerRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
if rt.handler != nil {
// Fake request with handler
rec := httptest.NewRecorder()
rt.handler.ServeHTTP(rec, req)
resp := rec.Result()
// Copy request to response
resp.Request = req
return resp, nil
}
return nil, errors.New("no handler")
}
func doHandlerRequest(req *http.Request, handler http.Handler) (*http.Response, error) {
client := &http.Client{
Transport: &handlerRoundTripper{handler: handler},
}
return client.Do(req)
}

4
utils_test.go

@ -79,6 +79,10 @@ func Test_urlHasExt(t *testing.T) {
func Test_cleanHTMLText(t *testing.T) {
assert.Equal(t, `"This is a 'test'" 😁`, cleanHTMLText(`"This is a 'test'" 😁`))
assert.Equal(t, `Test`, cleanHTMLText(`<b>Test</b>`))
assert.Equal(t, "Test\n\nTest", cleanHTMLText(`<p>Test</p><p>Test</p>`))
assert.Equal(t, "Test\n\nTest", cleanHTMLText("<p>Test</p>\n<p>Test</p>"))
assert.Equal(t, "Test\n\nTest", cleanHTMLText("<div><p>Test</p>\n<p>Test</p></div>"))
assert.Equal(t, "Test test\n\nTest", cleanHTMLText(`<p>Test <b>test</b></p><p>Test</p>`))
}
func Test_containsStrings(t *testing.T) {

66
webmention.go

@ -25,6 +25,7 @@ type mention struct {
Source string
NewSource string
Target string
NewTarget string
Created int64
Title string
Content string
@ -52,7 +53,9 @@ func (a *goBlog) handleWebmention(w http.ResponseWriter, r *http.Request) {
a.serveError(w, r, err.Error(), http.StatusBadRequest)
return
}
if !strings.HasPrefix(m.Target, a.cfg.Server.PublicAddress) {
hasShortPrefix := a.cfg.Server.ShortPublicAddress != "" && strings.HasPrefix(m.Target, a.cfg.Server.ShortPublicAddress)
hasLongPrefix := strings.HasPrefix(m.Target, a.cfg.Server.PublicAddress)
if !hasShortPrefix && !hasLongPrefix {
a.debug("Webmention target not allowed:", m.Target)
a.serveError(w, r, "target not allowed", http.StatusBadRequest)
return
@ -89,9 +92,21 @@ func (a *goBlog) extractMention(r *http.Request) (*mention, error) {
}, nil
}
func (db *database) webmentionExists(source, target string) bool {
func (db *database) webmentionExists(m *mention) bool {
result := 0
row, err := db.queryRow("select exists(select 1 from webmentions where lowerx(source) = lowerx(@source) and lowerx(target) = lowerx(@target))", sql.Named("source", source), sql.Named("target", target))
row, err := db.queryRow(
`
select exists(
select 1
from webmentions
where
lowerx(source) in (lowerx(@source), lowerx(@newsource))
and lowerx(target) in (lowerx(@target), lowerx(@newtarget))
)
`,
sql.Named("source", m.Source), sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)),
sql.Named("target", m.Target), sql.Named("newtarget", defaultIfEmpty(m.NewTarget, m.Target)),
)
if err != nil {
return false
}
@ -126,17 +141,56 @@ func (db *database) insertWebmention(m *mention, status webmentionStatus) error
return err
}
func (db *database) deleteWebmention(id int) error {
func (db *database) updateWebmention(m *mention, newStatus webmentionStatus) error {
_, err := db.exec(`
update webmentions
set
source = @newsource,
target = @newtarget,
status = @status,
title = @title,
content = @content,
author = @author
where
lowerx(source) in (lowerx(@source), lowerx(@newsource2))
and lowerx(target) in (lowerx(@target), lowerx(@newtarget2))
`,
sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)),
sql.Named("newtarget", defaultIfEmpty(m.NewTarget, m.Target)),
sql.Named("status", newStatus),
sql.Named("title", m.Title),
sql.Named("content", m.Content),
sql.Named("author", m.Author),
sql.Named("source", m.Source),
sql.Named("newsource2", defaultIfEmpty(m.NewSource, m.Source)),
sql.Named("target", m.Target),
sql.Named("newtarget2", defaultIfEmpty(m.NewTarget, m.Target)),
)
return err
}
func (db *database) deleteWebmentionId(id int) error {
_, err := db.exec("delete from webmentions where id = @id", sql.Named("id", id))
return err
}
func (db *database) approveWebmention(id int) error {
func (db *database) deleteWebmention(m *mention) error {
_, err := db.exec(
"delete from webmentions where lowerx(source) in (lowerx(@source), lowerx(@newsource)) and lowerx(target) in (lowerx(@target), lowerx(@newtarget))",
sql.Named("source", m.Source),
sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)),
sql.Named("target", m.Target),
sql.Named("newtarget", defaultIfEmpty(m.NewTarget, m.Target)),
)
return err
}
func (db *database) approveWebmentionId(id int) error {
_, err := db.exec("update webmentions set status = ? where id = ?", webmentionStatusApproved, id)
return err
}
func (a *goBlog) reverifyWebmention(id int) error {
func (a *goBlog) reverifyWebmentionId(id int) error {
m, err := a.db.getWebmentions(&webmentionsRequestConfig{
id: id,
limit: 1,

6
webmentionAdmin.go

@ -120,11 +120,11 @@ func (a *goBlog) webmentionAdminAction(w http.ResponseWriter, r *http.Request) {
}
switch action {
case "delete":
err = a.db.deleteWebmention(id)
err = a.db.deleteWebmentionId(id)
case "approve":
err = a.db.approveWebmention(id)
err = a.db.approveWebmentionId(id)
case "reverify":
err = a.reverifyWebmention(id)
err = a.reverifyWebmentionId(id)
}
if err != nil {
a.serveError(w, r, err.Error(), http.StatusInternalServerError)

110
webmentionVerification.go

@ -2,20 +2,19 @@ package main
import (
"bytes"
"database/sql"
"encoding/gob"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/thoas/go-funk"
"go.goblog.app/app/pkgs/contenttype"
"willnorris.com/go/microformats"
)
@ -62,78 +61,72 @@ func (a *goBlog) queueMention(m *mention) error {
}
func (a *goBlog) verifyMention(m *mention) error {
// Do request
req, err := http.NewRequest(http.MethodGet, m.Source, nil)
// Request target
targetReq, err := http.NewRequest(http.MethodGet, m.Target, nil)
if err != nil {
return err
}
var resp *http.Response
if strings.HasPrefix(m.Source, a.cfg.Server.PublicAddress) {
rec := httptest.NewRecorder()
for a.d == nil {
// Server not yet started
time.Sleep(1 * time.Second)
targetReq.Header.Set("Accept", contenttype.HTMLUTF8)
setLoggedIn(targetReq, true)
targetResp, err := doHandlerRequest(targetReq, a.getAppRouter())
if err != nil {
return err
}
// Check if target has a valid status code
if targetResp.StatusCode != http.StatusOK {
return a.db.deleteWebmention(m)
}
// Check if target has a redirect
if respReq := targetResp.Request; respReq != nil {
if ru := respReq.URL; m.Target != ru.String() {
m.NewTarget = ru.String()
}
setLoggedIn(req, true)
a.d.ServeHTTP(rec, req)
resp = rec.Result()
}
// Request source
sourceReq, err := http.NewRequest(http.MethodGet, m.Source, nil)
if err != nil {
return err
}
sourceReq.Header.Set("Accept", contenttype.HTMLUTF8)
var sourceResp *http.Response
if strings.HasPrefix(m.Source, a.cfg.Server.PublicAddress) ||
(a.cfg.Server.ShortPublicAddress != "" && strings.HasPrefix(m.Source, a.cfg.Server.ShortPublicAddress)) {
setLoggedIn(sourceReq, true)
sourceResp, err = doHandlerRequest(sourceReq, a.getAppRouter())
} else {
req.Header.Set(userAgent, appUserAgent)
resp, err = a.httpClient.Do(req)
if err != nil {
return err
}
sourceReq.Header.Set(userAgent, appUserAgent)
sourceResp, err = a.httpClient.Do(sourceReq)
}
if err != nil {
return err
}
// Check if source has a valid status code
if sourceResp.StatusCode != http.StatusOK {
return a.db.deleteWebmention(m)
}
// Check if source has a redirect
if respReq := resp.Request; respReq != nil {
if respReq := sourceResp.Request; respReq != nil {
if ru := respReq.URL; m.Source != ru.String() {
m.NewSource = ru.String()
}
}
// Parse response body
err = m.verifyReader(resp.Body)
_ = resp.Body.Close()
err = m.verifyReader(sourceResp.Body)
_ = sourceResp.Body.Close()
if err != nil {
// Delete webmentions with old or new source
_, err := a.db.exec(
"delete from webmentions where lowerx(source) in (lowerx(@source), lowerx(@newsource)) and lowerx(target) = lowerx(@target)",
sql.Named("source", m.Source),
sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)),
sql.Named("target", m.Target),
)
return err
return a.db.deleteWebmention(m)
}
if cr := []rune(m.Content); len(cr) > 500 {
m.Content = string(cr[0:497]) + "…"
}
m.Content = strings.ReplaceAll(m.Content, "\n", " ")
if tr := []rune(m.Title); len(tr) > 60 {
m.Title = string(tr[0:57]) + "…"
}
newStatus := webmentionStatusVerified
if a.db.webmentionExists(m.Source, m.Target) {
// Check if webmention also has webmention with new source
if m.NewSource != "" && a.db.webmentionExists(m.NewSource, m.Target) {
// Delete it
_, err = a.db.exec(
"delete from webmentions where lowerx(source) = lowerx(@source) and lowerx(target) = lowerx(@target)",
sql.Named("source", m.NewSource), sql.Named("target", m.Target),
)
if err != nil {
return err
}
}
// Update or insert webmention
if a.db.webmentionExists(m) {
// Update webmention
_, err = a.db.exec(
"update webmentions set source = @newsource, status = @status, title = @title, content = @content, author = @author where lowerx(source) = lowerx(@source) and lowerx(target) = lowerx(@target)",
sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)),
sql.Named("status", newStatus),
sql.Named("title", m.Title),
sql.Named("content", m.Content),
sql.Named("author", m.Author),
sql.Named("source", m.Source),
sql.Named("target", m.Target),
)
err = a.db.updateWebmention(m, newStatus)
if err != nil {
return err
}
@ -141,11 +134,14 @@ func (a *goBlog) verifyMention(m *mention) error {
if m.NewSource != "" {
m.Source = m.NewSource
}
if m.NewTarget != "" {
m.Target = m.NewTarget
}
err = a.db.insertWebmention(m, newStatus)
if err != nil {
return err
}
a.sendNotification(fmt.Sprintf("New webmention from %s to %s", m.Source, m.Target))
a.sendNotification(fmt.Sprintf("New webmention from %s to %s", defaultIfEmpty(m.NewSource, m.Source), defaultIfEmpty(m.NewTarget, m.Target)))
}
return err
}
@ -161,7 +157,7 @@ func (m *mention) verifyReader(body io.Reader) error {
return err
}
if _, hasLink := funk.FindString(links, func(s string) bool {
return unescapedPath(s) == unescapedPath(m.Target)
return unescapedPath(s) == unescapedPath(m.Target) || unescapedPath(s) == unescapedPath(m.NewTarget)
}); !hasLink {
return errors.New("target not found in source")
}
@ -233,6 +229,12 @@ func (m *mention) fillContent(mf *microformats.Microformat) {
if content, ok := contents[0].(map[string]string); ok {
if contentHTML, ok := content["html"]; ok {
m.Content = cleanHTMLText(contentHTML)
// Replace newlines with spaces
m.Content = strings.ReplaceAll(m.Content, "\n", " ")
// Collapse double spaces
m.Content = strings.Join(strings.Fields(m.Content), " ")
// Trim spaces
m.Content = strings.TrimSpace(m.Content)
}
}
}

8
webmentionVerification_test.go

@ -4,6 +4,7 @@ import (
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/require"
@ -28,6 +29,11 @@ func Test_verifyMention(t *testing.T) {
PublicAddress: "https://example.org",
},
},
d: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, r, r.URL.Path[:len(r.URL.Path)-1], http.StatusFound)
}
}),
}
_ = app.initDatabase(false)
@ -35,7 +41,7 @@ func Test_verifyMention(t *testing.T) {
m := &mention{
Source: "https://example.net/articles/micropub-crossposting-to-twitter-and-enabling-tweetstorms",
Target: "https://example.org/articles/micropub-syndication-targets-and-crossposting-to-mastodon",
Target: "https://example.org/articles/micropub-syndication-targets-and-crossposting-to-mastodon/",
}
err = app.verifyMention(m)

4
webmention_test.go

@ -52,7 +52,7 @@ func Test_webmentions(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, 1, count)
exists := app.db.webmentionExists("Https://Example.net/test", "Https://Example.com/TÄst")
exists := app.db.webmentionExists(&mention{Source: "Https://Example.net/test", Target: "Https://Example.com/TÄst"})
assert.True(t, exists)
mentions = app.db.getWebmentionsByAddress("https://example.com/täst")
@ -63,7 +63,7 @@ func Test_webmentions(t *testing.T) {
})
require.NoError(t, err)
if assert.Len(t, mentions, 1) {
_ = app.db.approveWebmention(mentions[0].ID)
_ = app.db.approveWebmentionId(mentions[0].ID)
}
mentions = app.db.getWebmentionsByAddress("https://example.com/täst")

Loading…
Cancel
Save