I’ve previously talked about how I crosspost from this blog to my Mastodon account without the need for a third-party service, and how I leverage WordPress’s hook system to even enable toot threading.
-In this post, I’m going to really quickly explain my (extremely similar) Twitter setup. (Note: I don’t actually syndicate this blog’s posts to Twitter, but I do use this very setup on another site of mine.)
+I’ve previously talked about how I crosspost from this blog to my Mastodon account without the need for a third-party service, and how I leverage WordPress’s hook system to even enable toot threading.
In this post, I’m going to really quickly explain my (extremely similar) Twitter setup. (Note: I don’t actually syndicate this blog’s posts to Twitter, but I do use this very setup on another site of mine.)
I liked the idea of a dead-simple Twitter plugin, so I forked my Mastodon plugin and tweaked a few things here and there. Once I’ve installed it, and created a developer account, generated the necessary keys, and let WordPress know about them, things look, well, very familiar. In fact, crossposting should now just work.
Now, to enable this when posting through Micropub rather than WordPress’s admin interface! Again, since posting through Micropub means no WordPress interface, and thus no “meta box” and no checkbox, and no way for WordPress to know if I wanted to crosspost a certain article or not, I’m going to have to use … syndication targets (which were invented precisely for this reason).
Test
Test
`)) + assert.Equal(t, "Test\n\nTest", cleanHTMLText("Test
\nTest
")) + assert.Equal(t, "Test\n\nTest", cleanHTMLText("Test
\nTest
Test test
Test
`)) } func Test_containsStrings(t *testing.T) { diff --git a/webmention.go b/webmention.go index 3b4069a..ffaafa1 100644 --- a/webmention.go +++ b/webmention.go @@ -25,6 +25,7 @@ type mention struct { Source string NewSource string Target string + NewTarget string Created int64 Title string Content string @@ -52,7 +53,9 @@ func (a *goBlog) handleWebmention(w http.ResponseWriter, r *http.Request) { a.serveError(w, r, err.Error(), http.StatusBadRequest) return } - if !strings.HasPrefix(m.Target, a.cfg.Server.PublicAddress) { + hasShortPrefix := a.cfg.Server.ShortPublicAddress != "" && strings.HasPrefix(m.Target, a.cfg.Server.ShortPublicAddress) + hasLongPrefix := strings.HasPrefix(m.Target, a.cfg.Server.PublicAddress) + if !hasShortPrefix && !hasLongPrefix { a.debug("Webmention target not allowed:", m.Target) a.serveError(w, r, "target not allowed", http.StatusBadRequest) return @@ -89,9 +92,21 @@ func (a *goBlog) extractMention(r *http.Request) (*mention, error) { }, nil } -func (db *database) webmentionExists(source, target string) bool { +func (db *database) webmentionExists(m *mention) bool { result := 0 - row, err := db.queryRow("select exists(select 1 from webmentions where lowerx(source) = lowerx(@source) and lowerx(target) = lowerx(@target))", sql.Named("source", source), sql.Named("target", target)) + row, err := db.queryRow( + ` + select exists( + select 1 + from webmentions + where + lowerx(source) in (lowerx(@source), lowerx(@newsource)) + and lowerx(target) in (lowerx(@target), lowerx(@newtarget)) + ) + `, + sql.Named("source", m.Source), sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)), + sql.Named("target", m.Target), sql.Named("newtarget", defaultIfEmpty(m.NewTarget, m.Target)), + ) if err != nil { return false } @@ -126,17 +141,56 @@ func (db *database) insertWebmention(m *mention, status webmentionStatus) error return err } -func (db *database) deleteWebmention(id int) error { +func (db *database) updateWebmention(m *mention, newStatus webmentionStatus) error { + _, err := db.exec(` + update webmentions + set + source = @newsource, + target = @newtarget, + status = @status, + title = @title, + content = @content, + author = @author + where + lowerx(source) in (lowerx(@source), lowerx(@newsource2)) + and lowerx(target) in (lowerx(@target), lowerx(@newtarget2)) + `, + sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)), + sql.Named("newtarget", defaultIfEmpty(m.NewTarget, m.Target)), + sql.Named("status", newStatus), + sql.Named("title", m.Title), + sql.Named("content", m.Content), + sql.Named("author", m.Author), + sql.Named("source", m.Source), + sql.Named("newsource2", defaultIfEmpty(m.NewSource, m.Source)), + sql.Named("target", m.Target), + sql.Named("newtarget2", defaultIfEmpty(m.NewTarget, m.Target)), + ) + return err +} + +func (db *database) deleteWebmentionId(id int) error { _, err := db.exec("delete from webmentions where id = @id", sql.Named("id", id)) return err } -func (db *database) approveWebmention(id int) error { +func (db *database) deleteWebmention(m *mention) error { + _, err := db.exec( + "delete from webmentions where lowerx(source) in (lowerx(@source), lowerx(@newsource)) and lowerx(target) in (lowerx(@target), lowerx(@newtarget))", + sql.Named("source", m.Source), + sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)), + sql.Named("target", m.Target), + sql.Named("newtarget", defaultIfEmpty(m.NewTarget, m.Target)), + ) + return err +} + +func (db *database) approveWebmentionId(id int) error { _, err := db.exec("update webmentions set status = ? where id = ?", webmentionStatusApproved, id) return err } -func (a *goBlog) reverifyWebmention(id int) error { +func (a *goBlog) reverifyWebmentionId(id int) error { m, err := a.db.getWebmentions(&webmentionsRequestConfig{ id: id, limit: 1, diff --git a/webmentionAdmin.go b/webmentionAdmin.go index 58e2362..4410150 100644 --- a/webmentionAdmin.go +++ b/webmentionAdmin.go @@ -120,11 +120,11 @@ func (a *goBlog) webmentionAdminAction(w http.ResponseWriter, r *http.Request) { } switch action { case "delete": - err = a.db.deleteWebmention(id) + err = a.db.deleteWebmentionId(id) case "approve": - err = a.db.approveWebmention(id) + err = a.db.approveWebmentionId(id) case "reverify": - err = a.reverifyWebmention(id) + err = a.reverifyWebmentionId(id) } if err != nil { a.serveError(w, r, err.Error(), http.StatusInternalServerError) diff --git a/webmentionVerification.go b/webmentionVerification.go index 03c59f2..8192b4c 100644 --- a/webmentionVerification.go +++ b/webmentionVerification.go @@ -2,20 +2,19 @@ package main import ( "bytes" - "database/sql" "encoding/gob" "errors" "fmt" "io" "log" "net/http" - "net/http/httptest" "net/url" "strings" "time" "github.com/PuerkitoBio/goquery" "github.com/thoas/go-funk" + "go.goblog.app/app/pkgs/contenttype" "willnorris.com/go/microformats" ) @@ -62,78 +61,72 @@ func (a *goBlog) queueMention(m *mention) error { } func (a *goBlog) verifyMention(m *mention) error { - // Do request - req, err := http.NewRequest(http.MethodGet, m.Source, nil) + // Request target + targetReq, err := http.NewRequest(http.MethodGet, m.Target, nil) if err != nil { return err } - var resp *http.Response - if strings.HasPrefix(m.Source, a.cfg.Server.PublicAddress) { - rec := httptest.NewRecorder() - for a.d == nil { - // Server not yet started - time.Sleep(1 * time.Second) - } - setLoggedIn(req, true) - a.d.ServeHTTP(rec, req) - resp = rec.Result() - } else { - req.Header.Set(userAgent, appUserAgent) - resp, err = a.httpClient.Do(req) - if err != nil { - return err + targetReq.Header.Set("Accept", contenttype.HTMLUTF8) + setLoggedIn(targetReq, true) + targetResp, err := doHandlerRequest(targetReq, a.getAppRouter()) + if err != nil { + return err + } + // Check if target has a valid status code + if targetResp.StatusCode != http.StatusOK { + return a.db.deleteWebmention(m) + } + // Check if target has a redirect + if respReq := targetResp.Request; respReq != nil { + if ru := respReq.URL; m.Target != ru.String() { + m.NewTarget = ru.String() } } + // Request source + sourceReq, err := http.NewRequest(http.MethodGet, m.Source, nil) + if err != nil { + return err + } + sourceReq.Header.Set("Accept", contenttype.HTMLUTF8) + var sourceResp *http.Response + if strings.HasPrefix(m.Source, a.cfg.Server.PublicAddress) || + (a.cfg.Server.ShortPublicAddress != "" && strings.HasPrefix(m.Source, a.cfg.Server.ShortPublicAddress)) { + setLoggedIn(sourceReq, true) + sourceResp, err = doHandlerRequest(sourceReq, a.getAppRouter()) + } else { + sourceReq.Header.Set(userAgent, appUserAgent) + sourceResp, err = a.httpClient.Do(sourceReq) + } + if err != nil { + return err + } + // Check if source has a valid status code + if sourceResp.StatusCode != http.StatusOK { + return a.db.deleteWebmention(m) + } // Check if source has a redirect - if respReq := resp.Request; respReq != nil { + if respReq := sourceResp.Request; respReq != nil { if ru := respReq.URL; m.Source != ru.String() { m.NewSource = ru.String() } } // Parse response body - err = m.verifyReader(resp.Body) - _ = resp.Body.Close() + err = m.verifyReader(sourceResp.Body) + _ = sourceResp.Body.Close() if err != nil { - // Delete webmentions with old or new source - _, err := a.db.exec( - "delete from webmentions where lowerx(source) in (lowerx(@source), lowerx(@newsource)) and lowerx(target) = lowerx(@target)", - sql.Named("source", m.Source), - sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)), - sql.Named("target", m.Target), - ) - return err + return a.db.deleteWebmention(m) } if cr := []rune(m.Content); len(cr) > 500 { m.Content = string(cr[0:497]) + "…" } - m.Content = strings.ReplaceAll(m.Content, "\n", " ") if tr := []rune(m.Title); len(tr) > 60 { m.Title = string(tr[0:57]) + "…" } newStatus := webmentionStatusVerified - if a.db.webmentionExists(m.Source, m.Target) { - // Check if webmention also has webmention with new source - if m.NewSource != "" && a.db.webmentionExists(m.NewSource, m.Target) { - // Delete it - _, err = a.db.exec( - "delete from webmentions where lowerx(source) = lowerx(@source) and lowerx(target) = lowerx(@target)", - sql.Named("source", m.NewSource), sql.Named("target", m.Target), - ) - if err != nil { - return err - } - } + // Update or insert webmention + if a.db.webmentionExists(m) { // Update webmention - _, err = a.db.exec( - "update webmentions set source = @newsource, status = @status, title = @title, content = @content, author = @author where lowerx(source) = lowerx(@source) and lowerx(target) = lowerx(@target)", - sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)), - sql.Named("status", newStatus), - sql.Named("title", m.Title), - sql.Named("content", m.Content), - sql.Named("author", m.Author), - sql.Named("source", m.Source), - sql.Named("target", m.Target), - ) + err = a.db.updateWebmention(m, newStatus) if err != nil { return err } @@ -141,11 +134,14 @@ func (a *goBlog) verifyMention(m *mention) error { if m.NewSource != "" { m.Source = m.NewSource } + if m.NewTarget != "" { + m.Target = m.NewTarget + } err = a.db.insertWebmention(m, newStatus) if err != nil { return err } - a.sendNotification(fmt.Sprintf("New webmention from %s to %s", m.Source, m.Target)) + a.sendNotification(fmt.Sprintf("New webmention from %s to %s", defaultIfEmpty(m.NewSource, m.Source), defaultIfEmpty(m.NewTarget, m.Target))) } return err } @@ -161,7 +157,7 @@ func (m *mention) verifyReader(body io.Reader) error { return err } if _, hasLink := funk.FindString(links, func(s string) bool { - return unescapedPath(s) == unescapedPath(m.Target) + return unescapedPath(s) == unescapedPath(m.Target) || unescapedPath(s) == unescapedPath(m.NewTarget) }); !hasLink { return errors.New("target not found in source") } @@ -233,6 +229,12 @@ func (m *mention) fillContent(mf *microformats.Microformat) { if content, ok := contents[0].(map[string]string); ok { if contentHTML, ok := content["html"]; ok { m.Content = cleanHTMLText(contentHTML) + // Replace newlines with spaces + m.Content = strings.ReplaceAll(m.Content, "\n", " ") + // Collapse double spaces + m.Content = strings.Join(strings.Fields(m.Content), " ") + // Trim spaces + m.Content = strings.TrimSpace(m.Content) } } } diff --git a/webmentionVerification_test.go b/webmentionVerification_test.go index a60066f..a142b3b 100644 --- a/webmentionVerification_test.go +++ b/webmentionVerification_test.go @@ -4,6 +4,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/require" @@ -28,6 +29,11 @@ func Test_verifyMention(t *testing.T) { PublicAddress: "https://example.org", }, }, + d: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/") { + http.Redirect(w, r, r.URL.Path[:len(r.URL.Path)-1], http.StatusFound) + } + }), } _ = app.initDatabase(false) @@ -35,7 +41,7 @@ func Test_verifyMention(t *testing.T) { m := &mention{ Source: "https://example.net/articles/micropub-crossposting-to-twitter-and-enabling-tweetstorms", - Target: "https://example.org/articles/micropub-syndication-targets-and-crossposting-to-mastodon", + Target: "https://example.org/articles/micropub-syndication-targets-and-crossposting-to-mastodon/", } err = app.verifyMention(m) diff --git a/webmention_test.go b/webmention_test.go index ae52bed..fb098dd 100644 --- a/webmention_test.go +++ b/webmention_test.go @@ -52,7 +52,7 @@ func Test_webmentions(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, count) - exists := app.db.webmentionExists("Https://Example.net/test", "Https://Example.com/TÄst") + exists := app.db.webmentionExists(&mention{Source: "Https://Example.net/test", Target: "Https://Example.com/TÄst"}) assert.True(t, exists) mentions = app.db.getWebmentionsByAddress("https://example.com/täst") @@ -63,7 +63,7 @@ func Test_webmentions(t *testing.T) { }) require.NoError(t, err) if assert.Len(t, mentions, 1) { - _ = app.db.approveWebmention(mentions[0].ID) + _ = app.db.approveWebmentionId(mentions[0].ID) } mentions = app.db.getWebmentionsByAddress("https://example.com/täst")