mirror of https://github.com/jlelse/GoBlog
Fix un-deletion for activitypub / mastodon, improve queue handling
This commit is contained in:
parent
c38c5e8ed9
commit
5f969f8ba8
|
@ -300,7 +300,7 @@ func (a *goBlog) apPost(p *post) {
|
|||
a.apSendToAllFollowers(p.Blog, map[string]any{
|
||||
"@context": []string{asContext},
|
||||
"actor": a.apIri(a.cfg.Blogs[p.Blog]),
|
||||
"id": a.fullPostURL(p),
|
||||
"id": a.activityPubId(p),
|
||||
"published": n.Published,
|
||||
"type": "Create",
|
||||
"object": n,
|
||||
|
@ -311,7 +311,7 @@ func (a *goBlog) apUpdate(p *post) {
|
|||
a.apSendToAllFollowers(p.Blog, map[string]any{
|
||||
"@context": []string{asContext},
|
||||
"actor": a.apIri(a.cfg.Blogs[p.Blog]),
|
||||
"id": a.fullPostURL(p),
|
||||
"id": a.activityPubId(p),
|
||||
"published": time.Now().Format("2006-01-02T15:04:05-07:00"),
|
||||
"type": "Update",
|
||||
"object": a.toASNote(p),
|
||||
|
@ -323,22 +323,22 @@ func (a *goBlog) apDelete(p *post) {
|
|||
"@context": []string{asContext},
|
||||
"actor": a.apIri(a.cfg.Blogs[p.Blog]),
|
||||
"type": "Delete",
|
||||
"object": a.fullPostURL(p),
|
||||
"object": a.activityPubId(p),
|
||||
})
|
||||
}
|
||||
|
||||
func (a *goBlog) apUndelete(p *post) {
|
||||
a.apSendToAllFollowers(p.Blog, map[string]any{
|
||||
"@context": []string{asContext},
|
||||
"actor": a.apIri(a.cfg.Blogs[p.Blog]),
|
||||
"type": "Undo",
|
||||
"object": map[string]any{
|
||||
"@context": []string{asContext},
|
||||
"actor": a.apIri(a.cfg.Blogs[p.Blog]),
|
||||
"type": "Delete",
|
||||
"object": a.fullPostURL(p),
|
||||
},
|
||||
})
|
||||
// The optimal way to do this would be to send a "Undo Delete" activity,
|
||||
// but that doesn't work with Mastodon yet.
|
||||
// see:
|
||||
// https://socialhub.activitypub.rocks/t/soft-deletes-and-restoring-deleted-posts/2318
|
||||
// https://github.com/mastodon/mastodon/issues/17553
|
||||
|
||||
// Update "activityPubVersion" parameter to current timestamp in nanoseconds
|
||||
p.Parameters[activityPubVersionParam] = []string{fmt.Sprintf("%d", utcNowNanos())}
|
||||
a.db.replacePostParam(p.Path, activityPubVersionParam, p.Parameters[activityPubVersionParam])
|
||||
// Post as new post
|
||||
a.apPost(p)
|
||||
}
|
||||
|
||||
func (a *goBlog) apAccept(blogName string, blog *configBlog, follow map[string]any) {
|
||||
|
@ -373,7 +373,7 @@ func (a *goBlog) apAccept(blogName string, blog *configBlog, follow map[string]a
|
|||
"object": follow,
|
||||
}
|
||||
_, accept["id"] = a.apNewID(blog)
|
||||
_ = a.db.apQueueSendSigned(a.apIri(blog), inbox, accept)
|
||||
_ = a.apQueueSendSigned(a.apIri(blog), inbox, accept)
|
||||
}
|
||||
|
||||
func (a *goBlog) apSendToAllFollowers(blog string, activity any) {
|
||||
|
@ -382,13 +382,13 @@ func (a *goBlog) apSendToAllFollowers(blog string, activity any) {
|
|||
log.Println("Failed to retrieve inboxes:", err.Error())
|
||||
return
|
||||
}
|
||||
a.db.apSendTo(a.apIri(a.cfg.Blogs[blog]), activity, inboxes)
|
||||
a.apSendTo(a.apIri(a.cfg.Blogs[blog]), activity, inboxes)
|
||||
}
|
||||
|
||||
func (db *database) apSendTo(blogIri string, activity any, inboxes []string) {
|
||||
func (a *goBlog) apSendTo(blogIri string, activity any, inboxes []string) {
|
||||
for _, i := range inboxes {
|
||||
go func(inbox string) {
|
||||
_ = db.apQueueSendSigned(blogIri, inbox, activity)
|
||||
_ = a.apQueueSendSigned(blogIri, inbox, activity)
|
||||
}(i)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -47,7 +47,7 @@ func (a *goBlog) initAPSendQueue() {
|
|||
})
|
||||
}
|
||||
|
||||
func (db *database) apQueueSendSigned(blogIri, to string, activity any) error {
|
||||
func (a *goBlog) apQueueSendSigned(blogIri, to string, activity any) error {
|
||||
body, err := json.Marshal(activity)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -61,7 +61,7 @@ func (db *database) apQueueSendSigned(blogIri, to string, activity any) error {
|
|||
}).encode(buf); err != nil {
|
||||
return err
|
||||
}
|
||||
return db.enqueue("ap", buf.Bytes(), time.Now())
|
||||
return a.enqueue("ap", buf.Bytes(), time.Now())
|
||||
}
|
||||
|
||||
func (r *apRequest) encode(w io.Writer) error {
|
||||
|
|
|
@ -101,7 +101,7 @@ func (a *goBlog) toASNote(p *post) *asNote {
|
|||
Context: []string{asContext},
|
||||
To: []string{"https://www.w3.org/ns/activitystreams#Public"},
|
||||
MediaType: contenttype.HTML,
|
||||
ID: a.fullPostURL(p),
|
||||
ID: a.activityPubId(p),
|
||||
URL: a.fullPostURL(p),
|
||||
AttributedTo: a.apIri(a.cfg.Blogs[p.Blog]),
|
||||
}
|
||||
|
@ -152,6 +152,16 @@ func (a *goBlog) toASNote(p *post) *asNote {
|
|||
return as
|
||||
}
|
||||
|
||||
const activityPubVersionParam = "activitypubversion"
|
||||
|
||||
func (a *goBlog) activityPubId(p *post) string {
|
||||
fu := a.fullPostURL(p)
|
||||
if version := p.firstParameter(activityPubVersionParam); version != "" {
|
||||
return fu + "?activitypubversion=" + version
|
||||
}
|
||||
return fu
|
||||
}
|
||||
|
||||
func (a *goBlog) serveActivityStreams(blog string, w http.ResponseWriter, r *http.Request) {
|
||||
b := a.cfg.Blogs[blog]
|
||||
publicKeyDer, err := x509.MarshalPKIXPublicKey(&(a.apPrivateKey.PublicKey))
|
||||
|
|
2
app.go
2
app.go
|
@ -73,6 +73,8 @@ type goBlog struct {
|
|||
regexRedirects []*regexRedirect
|
||||
// Sessions
|
||||
loginSessions, captchaSessions *dbSessionStore
|
||||
// Queue triggers
|
||||
queueTriggers []chan struct{}
|
||||
// Shutdown
|
||||
shutdown shutdowner.Shutdowner
|
||||
// Template strings
|
||||
|
|
|
@ -3,6 +3,7 @@ package main
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"sort"
|
||||
|
@ -66,7 +67,7 @@ func (a *goBlog) serveBlogrollExport(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
w.Header().Set(contentType, contenttype.XMLUTF8)
|
||||
_, _ = opmlBuf.WriteTo(w)
|
||||
_, _ = io.Copy(w, opmlBuf)
|
||||
}
|
||||
|
||||
func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) {
|
||||
|
|
|
@ -5,7 +5,6 @@ import (
|
|||
"errors"
|
||||
"log"
|
||||
"math"
|
||||
"strings"
|
||||
|
||||
"github.com/tkrajina/gpxgo/gpx"
|
||||
"golang.org/x/text/language"
|
||||
|
@ -102,7 +101,7 @@ func trackParseGPX(gpxString string) (result *trackParseResult, err error) {
|
|||
points []*trackPoint
|
||||
}
|
||||
|
||||
result.gpxData, err = gpx.Parse(strings.NewReader(gpxString))
|
||||
result.gpxData, err = gpx.ParseString(gpxString)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
3
go.mod
3
go.mod
|
@ -50,7 +50,8 @@ require (
|
|||
github.com/spf13/viper v1.10.1
|
||||
github.com/stretchr/testify v1.7.1
|
||||
github.com/tdewolff/minify/v2 v2.10.0
|
||||
github.com/tkrajina/gpxgo v1.2.1
|
||||
// master
|
||||
github.com/tkrajina/gpxgo v1.2.2-0.20220217201249-321f19554eec
|
||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
|
||||
github.com/vcraescu/go-paginator v1.0.1-0.20201114172518-2cfc59fe05c2
|
||||
github.com/yuin/goldmark v1.4.11
|
||||
|
|
4
go.sum
4
go.sum
|
@ -438,8 +438,8 @@ github.com/tdewolff/parse/v2 v2.5.27/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1
|
|||
github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4=
|
||||
github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
|
||||
github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
|
||||
github.com/tkrajina/gpxgo v1.2.1 h1:MJJtT4Re5btDGg89brFDrUP3EWz+cBmyo8pQwV0ZOak=
|
||||
github.com/tkrajina/gpxgo v1.2.1/go.mod h1:795sjVRFo5wWyN6oOZp0RYienGGBJjpAlgOz2nCngA0=
|
||||
github.com/tkrajina/gpxgo v1.2.2-0.20220217201249-321f19554eec h1:o5aL1yX+/xzvK4QfZe/iDcCoRrFoOQ/Dn43jV/thKBM=
|
||||
github.com/tkrajina/gpxgo v1.2.2-0.20220217201249-321f19554eec/go.mod h1:795sjVRFo5wWyN6oOZp0RYienGGBJjpAlgOz2nCngA0=
|
||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
|
||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
|
||||
github.com/u-root/uio v0.0.0-20210528114334-82958018845c h1:BFvcl34IGnw8yvJi8hlqLFo9EshRInwWBs2M5fGWzQA=
|
||||
|
|
|
@ -81,7 +81,7 @@ func (a *goBlog) serveMicropubQuery(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
w.Header().Set(contentType, contenttype.JSONUTF8)
|
||||
_, _ = buf.WriteTo(w)
|
||||
_, _ = io.Copy(w, buf)
|
||||
}
|
||||
|
||||
func (a *goBlog) serveMicropubPost(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
|
@ -2,6 +2,7 @@ package main
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"go.goblog.app/app/pkgs/bufferpool"
|
||||
|
@ -25,7 +26,7 @@ func (a *goBlog) serveNodeInfoDiscover(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
w.Header().Set(contentType, contenttype.JSONUTF8)
|
||||
mw := a.min.Writer(contenttype.JSON, w)
|
||||
_, _ = buf.WriteTo(mw)
|
||||
_, _ = io.Copy(mw, buf)
|
||||
_ = mw.Close()
|
||||
}
|
||||
|
||||
|
@ -60,6 +61,6 @@ func (a *goBlog) serveNodeInfo(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
w.Header().Set(contentType, contenttype.JSONUTF8)
|
||||
mw := a.min.Writer(contenttype.JSON, w)
|
||||
_, _ = buf.WriteTo(mw)
|
||||
_, _ = io.Copy(mw, buf)
|
||||
_ = mw.Close()
|
||||
}
|
||||
|
|
|
@ -74,6 +74,6 @@ func MergeMP3(out io.Writer, in ...io.Reader) error {
|
|||
}
|
||||
|
||||
// Copy the temporary output to the output
|
||||
_, err := tmpOut.WriteTo(out)
|
||||
_, err := io.Copy(out, tmpOut)
|
||||
return err
|
||||
}
|
||||
|
|
40
queue.go
40
queue.go
|
@ -18,21 +18,28 @@ type queueItem struct {
|
|||
id int
|
||||
}
|
||||
|
||||
func (db *database) enqueue(name string, content []byte, schedule time.Time) error {
|
||||
func (a *goBlog) enqueue(name string, content []byte, schedule time.Time) error {
|
||||
if len(content) == 0 {
|
||||
return errors.New("empty content")
|
||||
}
|
||||
_, err := db.exec(
|
||||
_, err := a.db.exec(
|
||||
"insert into queue (name, content, schedule) values (@name, @content, @schedule)",
|
||||
sql.Named("name", name),
|
||||
sql.Named("content", content),
|
||||
sql.Named("schedule", schedule.UTC().Format(time.RFC3339Nano)),
|
||||
)
|
||||
return err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Trigger all queue listeners
|
||||
for _, trigger := range a.queueTriggers {
|
||||
trigger <- struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *database) reschedule(qi *queueItem, dur time.Duration) error {
|
||||
_, err := db.exec(
|
||||
func (a *goBlog) reschedule(qi *queueItem, dur time.Duration) error {
|
||||
_, err := a.db.exec(
|
||||
"update queue set schedule = @schedule, content = @content where id = @id",
|
||||
sql.Named("schedule", qi.schedule.Add(dur).UTC().Format(time.RFC3339Nano)),
|
||||
sql.Named("content", qi.content),
|
||||
|
@ -41,13 +48,13 @@ func (db *database) reschedule(qi *queueItem, dur time.Duration) error {
|
|||
return err
|
||||
}
|
||||
|
||||
func (db *database) dequeue(qi *queueItem) error {
|
||||
_, err := db.exec("delete from queue where id = @id", sql.Named("id", qi.id))
|
||||
func (a *goBlog) dequeue(qi *queueItem) error {
|
||||
_, err := a.db.exec("delete from queue where id = @id", sql.Named("id", qi.id))
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *database) peekQueue(ctx context.Context, name string) (*queueItem, error) {
|
||||
row, err := db.queryRowContext(
|
||||
func (a *goBlog) peekQueue(ctx context.Context, name string) (*queueItem, error) {
|
||||
row, err := a.db.queryRowContext(
|
||||
ctx,
|
||||
"select id, name, content, schedule from queue where schedule <= @schedule and name = @name order by schedule asc limit 1",
|
||||
sql.Named("name", name),
|
||||
|
@ -75,6 +82,10 @@ func (db *database) peekQueue(ctx context.Context, name string) (*queueItem, err
|
|||
type queueProcessFunc func(qi *queueItem, dequeue func(), reschedule func(time.Duration))
|
||||
|
||||
func (a *goBlog) listenOnQueue(queueName string, wait time.Duration, process queueProcessFunc) {
|
||||
// Queue trigger
|
||||
trigger := make(chan struct{})
|
||||
a.queueTriggers = append(a.queueTriggers, trigger)
|
||||
// Start goroutine to listen on queue
|
||||
go func() {
|
||||
done := false
|
||||
var wg sync.WaitGroup
|
||||
|
@ -88,9 +99,9 @@ func (a *goBlog) listenOnQueue(queueName string, wait time.Duration, process que
|
|||
log.Println("Stopped queue:", queueName)
|
||||
})
|
||||
for !done {
|
||||
qi, err := a.db.peekQueue(ctx, queueName)
|
||||
qi, err := a.peekQueue(ctx, queueName)
|
||||
if err != nil {
|
||||
log.Println("queue error:", err.Error())
|
||||
log.Println("queue peek error:", err.Error())
|
||||
continue
|
||||
}
|
||||
if qi == nil {
|
||||
|
@ -98,19 +109,22 @@ func (a *goBlog) listenOnQueue(queueName string, wait time.Duration, process que
|
|||
select {
|
||||
case <-time.After(wait):
|
||||
continue
|
||||
case <-trigger:
|
||||
continue
|
||||
case <-ctx.Done():
|
||||
done = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
process(
|
||||
qi,
|
||||
func() {
|
||||
if err := a.db.dequeue(qi); err != nil {
|
||||
if err := a.dequeue(qi); err != nil {
|
||||
log.Println("queue dequeue error:", err.Error())
|
||||
}
|
||||
},
|
||||
func(dur time.Duration) {
|
||||
if err := a.db.reschedule(qi, dur); err != nil {
|
||||
if err := a.reschedule(qi, dur); err != nil {
|
||||
log.Println("queue reschedule error:", err.Error())
|
||||
}
|
||||
},
|
||||
|
|
|
@ -20,47 +20,46 @@ func Test_queue(t *testing.T) {
|
|||
}
|
||||
_ = app.initDatabase(false)
|
||||
defer app.db.close()
|
||||
db := app.db
|
||||
|
||||
time1 := time.Now()
|
||||
|
||||
err := db.enqueue("test", []byte(""), time.Now())
|
||||
err := app.enqueue("test", []byte(""), time.Now())
|
||||
require.Error(t, err)
|
||||
|
||||
err = db.enqueue("test", []byte("1"), time1)
|
||||
err = app.enqueue("test", []byte("1"), time1)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = db.enqueue("test", []byte("2"), time.Now())
|
||||
err = app.enqueue("test", []byte("2"), time.Now())
|
||||
require.NoError(t, err)
|
||||
|
||||
qi, err := db.peekQueue(context.Background(), "abc")
|
||||
qi, err := app.peekQueue(context.Background(), "abc")
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, qi)
|
||||
|
||||
qi, err = db.peekQueue(context.Background(), "test")
|
||||
qi, err = app.peekQueue(context.Background(), "test")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, qi)
|
||||
require.Equal(t, []byte("1"), qi.content)
|
||||
require.Equal(t, time1.UTC(), qi.schedule.UTC())
|
||||
|
||||
err = db.reschedule(qi, 1*time.Second)
|
||||
err = app.reschedule(qi, 1*time.Second)
|
||||
require.NoError(t, err)
|
||||
|
||||
qi, err = db.peekQueue(context.Background(), "test")
|
||||
qi, err = app.peekQueue(context.Background(), "test")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, qi)
|
||||
require.Equal(t, []byte("2"), qi.content)
|
||||
|
||||
err = db.dequeue(qi)
|
||||
err = app.dequeue(qi)
|
||||
require.NoError(t, err)
|
||||
|
||||
qi, err = db.peekQueue(context.Background(), "test")
|
||||
qi, err = app.peekQueue(context.Background(), "test")
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, qi)
|
||||
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
qi, err = db.peekQueue(context.Background(), "test")
|
||||
qi, err = app.peekQueue(context.Background(), "test")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, qi)
|
||||
require.Equal(t, []byte("1"), qi.content)
|
||||
|
@ -77,20 +76,19 @@ func Benchmark_queue(b *testing.B) {
|
|||
}
|
||||
_ = app.initDatabase(false)
|
||||
defer app.db.close()
|
||||
db := app.db
|
||||
|
||||
err := db.enqueue("test", []byte("1"), time.Now())
|
||||
err := app.enqueue("test", []byte("1"), time.Now())
|
||||
require.NoError(b, err)
|
||||
|
||||
b.Run("Peek with item", func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = db.peekQueue(context.Background(), "test")
|
||||
_, _ = app.peekQueue(context.Background(), "test")
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("Peek without item", func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = db.peekQueue(context.Background(), "abc")
|
||||
_, _ = app.peekQueue(context.Background(), "abc")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
4
utils.go
4
utils.go
|
@ -193,6 +193,10 @@ func utcNowString() string {
|
|||
return time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func utcNowNanos() int64 {
|
||||
return time.Now().UTC().UnixNano()
|
||||
}
|
||||
|
||||
type stringPair struct {
|
||||
First, Second string
|
||||
}
|
||||
|
|
|
@ -144,3 +144,13 @@ func Test_lowerUnescaptedPath(t *testing.T) {
|
|||
assert.Equal(t, "/de/posts/fahrradanhänger", lowerUnescapedPath("/de/posts/fahrradanh%C3%84nger"))
|
||||
assert.Equal(t, "/de/posts/fahrradanhänger", lowerUnescapedPath("/de/posts/fahrradanhÄnger"))
|
||||
}
|
||||
|
||||
func Fuzz_lowerUnescaptedPath(f *testing.F) {
|
||||
f.Add("/de/posts/fahrradanh%C3%84nger")
|
||||
f.Fuzz(func(t *testing.T, str string) {
|
||||
out := lowerUnescapedPath(str)
|
||||
if out == "" {
|
||||
t.Error("Empty output")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
@ -44,7 +44,7 @@ func (a *goBlog) queueMention(m *mention) error {
|
|||
if err := gob.NewEncoder(buf).Encode(m); err != nil {
|
||||
return err
|
||||
}
|
||||
return a.db.enqueue("wm", buf.Bytes(), time.Now())
|
||||
return a.enqueue("wm", buf.Bytes(), time.Now())
|
||||
}
|
||||
|
||||
func (a *goBlog) verifyMention(m *mention) error {
|
||||
|
|
|
@ -76,10 +76,11 @@ func Test_webmentions(t *testing.T) {
|
|||
mentions = app.db.getWebmentionsByAddress("https://example.com/t%C3%A4st")
|
||||
assert.Len(t, mentions, 1)
|
||||
|
||||
app.db.deleteWebmention(&mention{
|
||||
err = app.db.deleteWebmention(&mention{
|
||||
Source: "https://example.net/test",
|
||||
Target: "https://example.com/T%C3%84ST",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
mentions = app.db.getWebmentionsByAddress("https://example.com/täst")
|
||||
assert.Len(t, mentions, 0)
|
||||
|
|
Loading…
Reference in New Issue