Fix webmention

database.go

@@ -67,13 +67,14 @@ func (a *goBlog) openDatabase(file string, logging bool) (*database, error) {
 		ConnectHook: func(c *sqlite.SQLiteConn) error {
 			// Register functions
 			for n, f := range map[string]any{
-				"mdtext":    a.renderText,
-				"tolocal":   toLocalSafe,
-				"toutc":     toUTCSafe,
-				"wordcount": wordCount,
-				"charcount": charCount,
-				"urlize":    urlize,
-				"lowerx":    strings.ToLower,
+				"mdtext":         a.renderText,
+				"tolocal":        toLocalSafe,
+				"toutc":          toUTCSafe,
+				"wordcount":      wordCount,
+				"charcount":      charCount,
+				"urlize":         urlize,
+				"lowerx":         strings.ToLower,
+				"lowerunescaped": lowerUnescapedPath,
 			} {
 				if err := c.RegisterFunc(n, f, true); err != nil {
 					return err

go.mod

@@ -57,7 +57,7 @@ require (
 	// master
 	github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38
 	golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
+	golang.org/x/net v0.0.0-20220325170049-de3da57026de
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/text v0.3.7
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b

go.sum

@@ -575,8 +575,8 @@ golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc=
+golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

utils.go

@@ -116,6 +116,10 @@ func unescapedPath(p string) string {
 	return p
 }
 
+func lowerUnescapedPath(p string) string {
+	return strings.ToLower(unescapedPath(p))
+}
+
 type stringGroup struct {
 	Identifier string
 	Strings    []string

utils_test.go

@@ -134,3 +134,13 @@ func Test_matchTimeDiffLocale(t *testing.T) {
 	assert.Equal(t, "pt", string(matchTimeDiffLocale("pt-BR")))
 	assert.Equal(t, "pt", string(matchTimeDiffLocale("pt")))
 }
+
+func Test_unescapedPath(t *testing.T) {
+	assert.Equal(t, "/de/posts/fahrradanhänger", unescapedPath("/de/posts/fahrradanh%C3%A4nger"))
+	assert.Equal(t, "/de/posts/fahrradanhänger", unescapedPath("/de/posts/fahrradanhänger"))
+}
+
+func Test_lowerUnescaptedPath(t *testing.T) {
+	assert.Equal(t, "/de/posts/fahrradanhänger", lowerUnescapedPath("/de/posts/fahrradanh%C3%84nger"))
+	assert.Equal(t, "/de/posts/fahrradanhänger", lowerUnescapedPath("/de/posts/fahrradanhÄnger"))
+}

webmention.go

@@ -89,7 +89,7 @@ func (a *goBlog) extractMention(r *http.Request) (*mention, error) {
 		return nil, err
 	}
 	source := r.Form.Get("source")
-	target := unescapedPath(r.Form.Get("target"))
+	target := r.Form.Get("target")
 	if source == "" || target == "" || !isAbsoluteURL(source) || !isAbsoluteURL(target) {
 		a.debug("Invalid webmention request, source:", source, "target:", target)
 		return nil, errors.New("invalid request")
@@ -109,8 +109,8 @@ func (db *database) webmentionExists(m *mention) bool {
 			select 1
 			from webmentions
 			where
-				lowerx(source) in (lowerx(@source), lowerx(@newsource))
-				and lowerx(target) in (lowerx(@target), lowerx(@newtarget))
+				lowerunescaped(source) in (lowerunescaped(@source), lowerunescaped(@newsource))
+				and lowerunescaped(target) in (lowerunescaped(@target), lowerunescaped(@newtarget))
 		)
 		`,
 		sql.Named("source", m.Source), sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)),
@@ -128,7 +128,7 @@ func (db *database) webmentionExists(m *mention) bool {
 func (a *goBlog) createWebmention(source, target string) (err error) {
 	return a.queueMention(&mention{
 		Source:  source,
-		Target:  unescapedPath(target),
+		Target:  target,
 		Created: time.Now().Unix(),
 	})
 }
@@ -137,7 +137,7 @@ func (db *database) insertWebmention(m *mention, status webmentionStatus) error
 	_, err := db.exec(
 		`
 		insert into webmentions (source, target, url, created, status, title, content, author) 
-		values (@source, @target, @url, @created, @status, @title, @content, @author)
+		values (@source, lowerunescaped(@target), @url, @created, @status, @title, @content, @author)
 		`,
 		sql.Named("source", m.Source),
 		sql.Named("target", m.Target),
@@ -156,15 +156,15 @@ func (db *database) updateWebmention(m *mention, newStatus webmentionStatus) err
 			update webmentions
 			set 
 				source = @newsource,
-				target = @newtarget,
+				target = lowerunescaped(@newtarget),
 				url = @url,
 				status = @status,
 				title = @title,
 				content = @content,
 				author = @author
 			where
-				lowerx(source) in (lowerx(@source), lowerx(@newsource2))
-				and lowerx(target) in (lowerx(@target), lowerx(@newtarget2))
+				lowerunescaped(source) in (lowerunescaped(@source), lowerunescaped(@newsource2))
+				and lowerunescaped(target) in (lowerunescaped(@target), lowerunescaped(@newtarget2))
 			`,
 		sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)),
 		sql.Named("newtarget", defaultIfEmpty(m.NewTarget, m.Target)),
@@ -188,7 +188,7 @@ func (db *database) deleteWebmentionId(id int) error {
 
 func (db *database) deleteWebmention(m *mention) error {
 	_, err := db.exec(
-		"delete from webmentions where lowerx(source) in (lowerx(@source), lowerx(@newsource)) and lowerx(target) in (lowerx(@target), lowerx(@newtarget))",
+		"delete from webmentions where lowerunescaped(source) in (lowerunescaped(@source), lowerunescaped(@newsource)) and lowerunescaped(target) in (lowerunescaped(@target), lowerunescaped(@newtarget))",
 		sql.Named("source", m.Source),
 		sql.Named("newsource", defaultIfEmpty(m.NewSource, m.Source)),
 		sql.Named("target", m.Target),
@@ -233,7 +233,7 @@ func buildWebmentionsQuery(config *webmentionsRequestConfig) (query string, args
 	if config != nil {
 		queryBuilder.WriteString("where 1")
 		if config.target != "" {
-			queryBuilder.WriteString(" and lowerx(target) = lowerx(@target)")
+			queryBuilder.WriteString(" and lowerunescaped(target) = lowerunescaped(@target)")
 			args = append(args, sql.Named("target", config.target))
 		}
 		if config.status != "" {
@@ -241,7 +241,7 @@ func buildWebmentionsQuery(config *webmentionsRequestConfig) (query string, args
 			args = append(args, sql.Named("status", config.status))
 		}
 		if config.sourcelike != "" {
-			queryBuilder.WriteString(" and lowerx(source) like ('%' || lowerx(@sourcelike) || '%')")
+			queryBuilder.WriteString(" and lowerunescaped(source) like ('%' || lowerunescaped(@sourcelike) || '%')")
 			args = append(args, sql.Named("sourcelike", config.sourcelike))
 		}
 		if config.id != 0 {

webmentionVerification.go

@@ -180,7 +180,7 @@ func (a *goBlog) verifyReader(m *mention, body io.Reader) error {
 			return false
 		}
 		_ = resp.Body.Close()
-		if resp.StatusCode == http.StatusOK && unescapedPath(resp.Request.URL.String()) == unescapedPath(defaultIfEmpty(m.NewTarget, m.Target)) {
+		if resp.StatusCode == http.StatusOK && lowerUnescapedPath(resp.Request.URL.String()) == lowerUnescapedPath(defaultIfEmpty(m.NewTarget, m.Target)) {
 			return true
 		}
 		return false

webmention_test.go

@@ -73,4 +73,15 @@ func Test_webmentions(t *testing.T) {
 	mentions = app.db.getWebmentionsByAddress("https://example.com/täst")
 	assert.Len(t, mentions, 1)
 
+	mentions = app.db.getWebmentionsByAddress("https://example.com/t%C3%A4st")
+	assert.Len(t, mentions, 1)
+
+	app.db.deleteWebmention(&mention{
+		Source: "https://example.net/test",
+		Target: "https://example.com/T%C3%84ST",
+	})
+
+	mentions = app.db.getWebmentionsByAddress("https://example.com/täst")
+	assert.Len(t, mentions, 0)
+
 }