2020-11-16 11:05:34 +00:00
package main
import (
"bytes"
2020-11-16 17:34:29 +00:00
"database/sql"
2020-11-16 11:05:34 +00:00
"errors"
2020-11-16 17:34:29 +00:00
"fmt"
2020-11-16 11:05:34 +00:00
"io"
2020-11-25 11:36:14 +00:00
"log"
2020-11-16 11:05:34 +00:00
"net/http"
"net/url"
2020-11-25 11:36:14 +00:00
"os"
2020-11-16 17:34:29 +00:00
"strings"
2020-11-16 11:05:34 +00:00
2020-11-16 13:18:14 +00:00
"github.com/PuerkitoBio/goquery"
2020-11-25 11:36:14 +00:00
"github.com/joncrlsn/dque"
2020-11-16 11:05:34 +00:00
"willnorris.com/go/microformats"
)
2020-11-25 11:36:14 +00:00
var wmQueue * dque . DQue
func wmMentionBuilder ( ) interface { } {
return & mention { }
}
func initWebmentionQueue ( ) ( err error ) {
queuePath := "queues"
if _ , err := os . Stat ( queuePath ) ; os . IsNotExist ( err ) {
os . Mkdir ( queuePath , 0755 )
}
wmQueue , err = dque . NewOrOpen ( "webmention" , queuePath , 5 , wmMentionBuilder )
if err != nil {
return err
}
startWebmentionQueue ( )
return nil
}
func startWebmentionQueue ( ) {
2020-11-16 17:34:29 +00:00
go func ( ) {
for {
2020-11-25 11:36:14 +00:00
if i , err := wmQueue . PeekBlock ( ) ; err == nil {
if i == nil {
// Empty request
_ , _ = wmQueue . Dequeue ( )
continue
}
if m , ok := i . ( * mention ) ; ok {
err = m . verifyMention ( )
if err != nil {
log . Println ( fmt . Sprintf ( "Failed to verify webmention from %s to %s: %s" , m . Source , m . Target , err . Error ( ) ) )
}
_ , _ = wmQueue . Dequeue ( )
} else {
// Invalid type
_ , _ = wmQueue . Dequeue ( )
}
}
2020-11-16 11:05:34 +00:00
}
2020-11-16 17:34:29 +00:00
} ( )
}
2020-11-25 11:36:14 +00:00
func queueMention ( m * mention ) error {
return wmQueue . Enqueue ( m )
}
func ( m * mention ) verifyMention ( ) error {
req , err := http . NewRequest ( http . MethodGet , m . Source , nil )
2020-11-16 17:34:29 +00:00
if err != nil {
return err
}
2020-11-25 11:36:14 +00:00
req . Header . Set ( userAgent , appUserAgent )
resp , err := http . DefaultClient . Do ( req )
if err != nil {
2020-11-16 17:34:29 +00:00
return err
2020-11-16 11:05:34 +00:00
}
2020-11-25 11:36:14 +00:00
err = m . verifyReader ( resp . Body )
_ = resp . Body . Close ( )
if err != nil {
_ , err := appDbExec ( "delete from webmentions where source = @source and target = @target" , sql . Named ( "source" , m . Source ) , sql . Named ( "target" , m . Target ) )
return err
2020-11-16 17:34:29 +00:00
}
if len ( m . Content ) > 500 {
m . Content = m . Content [ 0 : 497 ] + "…"
}
2020-12-13 09:39:00 +00:00
if len ( m . Title ) > 60 {
m . Title = m . Title [ 0 : 57 ] + "…"
}
2020-11-16 17:34:29 +00:00
newStatus := webmentionStatusVerified
if strings . HasPrefix ( m . Source , appConfig . Server . PublicAddress ) {
// Approve if it's server-intern
newStatus = webmentionStatusApproved
}
2020-11-25 11:36:14 +00:00
if webmentionExists ( m . Source , m . Target ) {
_ , err = appDbExec ( "update webmentions set status = @status, title = @title, content = @content, author = @author where source = @source and target = @target" ,
sql . Named ( "status" , newStatus ) , sql . Named ( "title" , m . Title ) , sql . Named ( "content" , m . Content ) , sql . Named ( "author" , m . Author ) , sql . Named ( "source" , m . Source ) , sql . Named ( "target" , m . Target ) )
} else {
_ , err = appDbExec ( "insert into webmentions (source, target, created, status, title, content, author) values (@source, @target, @created, @status, @title, @content, @author)" ,
sql . Named ( "source" , m . Source ) , sql . Named ( "target" , m . Target ) , sql . Named ( "created" , m . Created ) , sql . Named ( "status" , newStatus ) , sql . Named ( "title" , m . Title ) , sql . Named ( "content" , m . Content ) , sql . Named ( "author" , m . Author ) )
2020-11-16 17:34:29 +00:00
sendNotification ( fmt . Sprintf ( "New webmention from %s to %s" , m . Source , m . Target ) )
}
return err
}
2020-11-25 11:36:14 +00:00
func ( m * mention ) verifyReader ( body io . Reader ) error {
2020-11-16 13:18:14 +00:00
var linksBuffer , gqBuffer , mfBuffer bytes . Buffer
io . Copy ( io . MultiWriter ( & linksBuffer , & gqBuffer , & mfBuffer ) , body )
// Check if source mentions target
links , err := allLinksFromHTML ( & linksBuffer , m . Source )
2020-11-16 11:05:34 +00:00
if err != nil {
return err
}
2020-11-16 13:18:14 +00:00
hasLink := false
for _ , link := range links {
2020-11-17 19:01:02 +00:00
if unescapedPath ( link ) == unescapedPath ( m . Target ) {
2020-11-16 13:18:14 +00:00
hasLink = true
break
2020-11-16 11:05:34 +00:00
}
}
2020-11-16 13:18:14 +00:00
if ! hasLink {
return errors . New ( "target not found in source" )
2020-11-16 11:05:34 +00:00
}
2020-11-16 13:18:14 +00:00
// Set title
doc , err := goquery . NewDocumentFromReader ( & gqBuffer )
if err != nil {
return err
}
if title := doc . Find ( "title" ) ; title != nil {
m . Title = title . Text ( )
}
// Fill mention attributes
sourceURL , err := url . Parse ( m . Source )
if err != nil {
return err
}
2020-11-25 11:36:14 +00:00
m . fillFromData ( microformats . Parse ( & mfBuffer , sourceURL ) )
2020-11-16 11:05:34 +00:00
return nil
}
2020-11-25 11:36:14 +00:00
func ( m * mention ) fillFromData ( mf * microformats . Data ) {
2020-11-16 11:05:34 +00:00
for _ , i := range mf . Items {
2020-11-25 11:36:14 +00:00
m . fill ( i )
2020-11-16 11:05:34 +00:00
}
}
2020-11-25 11:36:14 +00:00
func ( m * mention ) fill ( mf * microformats . Microformat ) bool {
2020-11-16 11:05:34 +00:00
if mfHasType ( mf , "h-entry" ) {
if name , ok := mf . Properties [ "name" ] ; ok && len ( name ) > 0 {
if title , ok := name [ 0 ] . ( string ) ; ok {
2020-11-16 13:18:14 +00:00
m . Title = title
2020-11-16 11:05:34 +00:00
}
}
if contents , ok := mf . Properties [ "content" ] ; ok && len ( contents ) > 0 {
2020-11-16 14:46:16 +00:00
if content , ok := contents [ 0 ] . ( map [ string ] string ) ; ok {
if contentValue , ok := content [ "value" ] ; ok {
m . Content = contentValue
2020-11-16 11:05:34 +00:00
}
}
}
if authors , ok := mf . Properties [ "author" ] ; ok && len ( authors ) > 0 {
if author , ok := authors [ 0 ] . ( * microformats . Microformat ) ; ok {
if names , ok := author . Properties [ "name" ] ; ok && len ( names ) > 0 {
2020-11-16 13:18:14 +00:00
if name , ok := names [ 0 ] . ( string ) ; ok {
m . Author = name
2020-11-16 11:05:34 +00:00
}
}
}
}
return true
} else if len ( mf . Children ) > 0 {
2020-11-16 13:18:14 +00:00
for _ , mfc := range mf . Children {
2020-11-25 11:36:14 +00:00
if m . fill ( mfc ) {
2020-11-16 11:05:34 +00:00
return true
}
}
}
return false
}
func mfHasType ( mf * microformats . Microformat , typ string ) bool {
for _ , t := range mf . Type {
if typ == t {
return true
}
}
return false
}