mirror of https://github.com/jlelse/GoBlog
More pooled buffers and don't AP announce replies
This commit is contained in:
parent
b453e6b400
commit
0f1408fe3e
|
@ -49,7 +49,9 @@ func (a *goBlog) initActivityPub() error {
|
||||||
a.apDelete(p)
|
a.apDelete(p)
|
||||||
})
|
})
|
||||||
a.pUndeleteHooks = append(a.pUndeleteHooks, func(p *post) {
|
a.pUndeleteHooks = append(a.pUndeleteHooks, func(p *post) {
|
||||||
a.apUndelete(p)
|
if p.isPublishedSectionPost() {
|
||||||
|
a.apUndelete(p)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
// Prepare webfinger
|
// Prepare webfinger
|
||||||
a.webfingerResources = map[string]*configBlog{}
|
a.webfingerResources = map[string]*configBlog{}
|
||||||
|
@ -185,7 +187,7 @@ func (a *goBlog) apHandleInbox(w http.ResponseWriter, r *http.Request) {
|
||||||
// May be a mention; find links to blog and save them as webmentions
|
// May be a mention; find links to blog and save them as webmentions
|
||||||
if links, err := allLinksFromHTMLString(content, baseUrl); err == nil {
|
if links, err := allLinksFromHTMLString(content, baseUrl); err == nil {
|
||||||
for _, link := range links {
|
for _, link := range links {
|
||||||
if strings.HasPrefix(link, blogIri) {
|
if strings.HasPrefix(link, a.cfg.Server.PublicAddress) {
|
||||||
_ = a.createWebmention(baseUrl, link)
|
_ = a.createWebmention(baseUrl, link)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -313,11 +315,6 @@ func (a *goBlog) apPost(p *post) {
|
||||||
"type": "Create",
|
"type": "Create",
|
||||||
"object": n,
|
"object": n,
|
||||||
})
|
})
|
||||||
if n.InReplyTo != "" {
|
|
||||||
// Is reply, so announce it
|
|
||||||
time.Sleep(30 * time.Second)
|
|
||||||
a.apAnnounce(p)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *goBlog) apUpdate(p *post) {
|
func (a *goBlog) apUpdate(p *post) {
|
||||||
|
@ -331,17 +328,6 @@ func (a *goBlog) apUpdate(p *post) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *goBlog) apAnnounce(p *post) {
|
|
||||||
a.apSendToAllFollowers(p.Blog, map[string]interface{}{
|
|
||||||
"@context": []string{asContext},
|
|
||||||
"actor": a.apIri(a.cfg.Blogs[p.Blog]),
|
|
||||||
"id": a.fullPostURL(p) + "#announce",
|
|
||||||
"published": a.toASNote(p).Published,
|
|
||||||
"type": "Announce",
|
|
||||||
"object": a.fullPostURL(p),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *goBlog) apDelete(p *post) {
|
func (a *goBlog) apDelete(p *post) {
|
||||||
a.apSendToAllFollowers(p.Blog, map[string]interface{}{
|
a.apSendToAllFollowers(p.Blog, map[string]interface{}{
|
||||||
"@context": []string{asContext},
|
"@context": []string{asContext},
|
||||||
|
@ -450,17 +436,16 @@ func (a *goBlog) loadActivityPubPrivateKey() error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Generate and cache key
|
// Generate and cache key
|
||||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
var err error
|
||||||
|
a.apPrivateKey, err = rsa.GenerateKey(rand.Reader, 2048)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
encodedKey := x509.MarshalPKCS1PrivateKey(key)
|
return a.db.cachePersistently(
|
||||||
pemEncoded := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: encodedKey})
|
"activitypub_key",
|
||||||
err = a.db.cachePersistently("activitypub_key", pemEncoded)
|
pem.EncodeToMemory(&pem.Block{
|
||||||
if err != nil {
|
Type: "PRIVATE KEY",
|
||||||
return err
|
Bytes: x509.MarshalPKCS1PrivateKey(a.apPrivateKey),
|
||||||
}
|
}),
|
||||||
a.apPrivateKey = key
|
)
|
||||||
// Return key
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,11 +6,14 @@ import (
|
||||||
"encoding/gob"
|
"encoding/gob"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"go.goblog.app/app/pkgs/bufferpool"
|
||||||
"go.goblog.app/app/pkgs/contenttype"
|
"go.goblog.app/app/pkgs/contenttype"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -22,40 +25,49 @@ type apRequest struct {
|
||||||
|
|
||||||
func (a *goBlog) initAPSendQueue() {
|
func (a *goBlog) initAPSendQueue() {
|
||||||
go func() {
|
go func() {
|
||||||
for {
|
done := false
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
|
a.shutdown.Add(func() {
|
||||||
|
done = true
|
||||||
|
wg.Wait()
|
||||||
|
log.Println("Stopped AP send queue")
|
||||||
|
})
|
||||||
|
for !done {
|
||||||
qi, err := a.db.peekQueue("ap")
|
qi, err := a.db.peekQueue("ap")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println("activitypub send queue:", err.Error())
|
log.Println("activitypub send queue:", err.Error())
|
||||||
continue
|
continue
|
||||||
} else if qi != nil {
|
}
|
||||||
var r apRequest
|
if qi == nil {
|
||||||
err = gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&r)
|
// No item in the queue, wait a moment
|
||||||
if err != nil {
|
time.Sleep(5 * time.Second)
|
||||||
log.Println("activitypub send queue:", err.Error())
|
continue
|
||||||
_ = a.db.dequeue(qi)
|
}
|
||||||
|
var r apRequest
|
||||||
|
if err = gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&r); err != nil {
|
||||||
|
log.Println("activitypub send queue:", err.Error())
|
||||||
|
_ = a.db.dequeue(qi)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err = a.apSendSigned(r.BlogIri, r.To, r.Activity); err != nil {
|
||||||
|
if r.Try++; r.Try < 20 {
|
||||||
|
// Try it again
|
||||||
|
buf := bufferpool.Get()
|
||||||
|
_ = r.encode(buf)
|
||||||
|
qi.content = buf.Bytes()
|
||||||
|
_ = a.db.reschedule(qi, time.Duration(r.Try)*10*time.Minute)
|
||||||
|
bufferpool.Put(buf)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err := a.apSendSigned(r.BlogIri, r.To, r.Activity); err != nil {
|
log.Println("AP request failed for the 20th time:", r.To)
|
||||||
if r.Try++; r.Try < 20 {
|
_ = a.db.apRemoveInbox(r.To)
|
||||||
// Try it again
|
}
|
||||||
qi.content, _ = r.encode()
|
if err = a.db.dequeue(qi); err != nil {
|
||||||
_ = a.db.reschedule(qi, time.Duration(r.Try)*10*time.Minute)
|
log.Println("activitypub send queue:", err.Error())
|
||||||
continue
|
|
||||||
} else {
|
|
||||||
log.Printf("Request to %s failed for the 20th time", r.To)
|
|
||||||
log.Println()
|
|
||||||
_ = a.db.apRemoveInbox(r.To)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
err = a.db.dequeue(qi)
|
|
||||||
if err != nil {
|
|
||||||
log.Println("activitypub send queue:", err.Error())
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// No item in the queue, wait a moment
|
|
||||||
time.Sleep(15 * time.Second)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
wg.Done()
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -64,24 +76,20 @@ func (db *database) apQueueSendSigned(blogIri, to string, activity interface{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
b, err := (&apRequest{
|
buf := bufferpool.Get()
|
||||||
|
defer bufferpool.Put(buf)
|
||||||
|
if err := (&apRequest{
|
||||||
BlogIri: blogIri,
|
BlogIri: blogIri,
|
||||||
To: to,
|
To: to,
|
||||||
Activity: body,
|
Activity: body,
|
||||||
}).encode()
|
}).encode(buf); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return db.enqueue("ap", b, time.Now())
|
return db.enqueue("ap", buf.Bytes(), time.Now())
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *apRequest) encode() ([]byte, error) {
|
func (r *apRequest) encode(w io.Writer) error {
|
||||||
var buf bytes.Buffer
|
return gob.NewEncoder(w).Encode(r)
|
||||||
err := gob.NewEncoder(&buf).Encode(r)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return buf.Bytes(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *goBlog) apSendSigned(blogIri, to string, activity []byte) error {
|
func (a *goBlog) apSendSigned(blogIri, to string, activity []byte) error {
|
||||||
|
@ -89,9 +97,7 @@ func (a *goBlog) apSendSigned(blogIri, to string, activity []byte) error {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
// Create request
|
// Create request
|
||||||
var requestBuffer bytes.Buffer
|
r, err := http.NewRequestWithContext(ctx, http.MethodPost, to, bytes.NewReader(activity))
|
||||||
requestBuffer.Write(activity)
|
|
||||||
r, err := http.NewRequestWithContext(ctx, http.MethodPost, to, &requestBuffer)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -117,7 +123,7 @@ func (a *goBlog) apSendSigned(blogIri, to string, activity []byte) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
_ = resp.Body.Close()
|
||||||
if !apRequestIsSuccess(resp.StatusCode) {
|
if !apRequestIsSuccess(resp.StatusCode) {
|
||||||
return fmt.Errorf("signed request failed with status %d", resp.StatusCode)
|
return fmt.Errorf("signed request failed with status %d", resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,7 +3,6 @@ package main
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"io"
|
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"sort"
|
"sort"
|
||||||
|
@ -67,7 +66,7 @@ func (a *goBlog) serveBlogrollExport(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Header().Set(contentType, contenttype.XMLUTF8)
|
w.Header().Set(contentType, contenttype.XMLUTF8)
|
||||||
_, _ = io.Copy(w, opmlBuf)
|
_, _ = opmlBuf.WriteTo(w)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) {
|
func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) {
|
||||||
|
@ -108,13 +107,14 @@ func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *database) cacheOutlines(blog string, outlines []*opml.Outline) {
|
func (db *database) cacheOutlines(blog string, outlines []*opml.Outline) {
|
||||||
var opmlBuffer bytes.Buffer
|
opmlBuffer := bufferpool.Get()
|
||||||
_ = opml.Render(&opmlBuffer, &opml.OPML{
|
_ = opml.Render(opmlBuffer, &opml.OPML{
|
||||||
Version: "2.0",
|
Version: "2.0",
|
||||||
DateCreated: time.Now().UTC(),
|
DateCreated: time.Now().UTC(),
|
||||||
Outlines: outlines,
|
Outlines: outlines,
|
||||||
})
|
})
|
||||||
_ = db.cachePersistently("blogroll_"+blog, opmlBuffer.Bytes())
|
_ = db.cachePersistently("blogroll_"+blog, opmlBuffer.Bytes())
|
||||||
|
bufferpool.Put(opmlBuffer)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *database) loadOutlineCache(blog string) []*opml.Outline {
|
func (db *database) loadOutlineCache(blog string) []*opml.Outline {
|
||||||
|
|
|
@ -5,6 +5,8 @@ import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/gob"
|
"encoding/gob"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
|
"go.goblog.app/app/pkgs/bufferpool"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -195,9 +197,10 @@ func (db *database) getBlogStats(blog string) (data *blogStatsData, err error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *database) cacheBlogStats(blog string, stats *blogStatsData) {
|
func (db *database) cacheBlogStats(blog string, stats *blogStatsData) {
|
||||||
var buf bytes.Buffer
|
buf := bufferpool.Get()
|
||||||
_ = gob.NewEncoder(&buf).Encode(stats)
|
_ = gob.NewEncoder(buf).Encode(stats)
|
||||||
_ = db.cachePersistently("blogstats_"+blog, buf.Bytes())
|
_ = db.cachePersistently("blogstats_"+blog, buf.Bytes())
|
||||||
|
bufferpool.Put(buf)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *database) loadBlogStatsCache(blog string) (stats *blogStatsData) {
|
func (db *database) loadBlogStatsCache(blog string) (stats *blogStatsData) {
|
||||||
|
|
30
contact.go
30
contact.go
|
@ -1,7 +1,6 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
@ -10,6 +9,7 @@ import (
|
||||||
|
|
||||||
"github.com/emersion/go-sasl"
|
"github.com/emersion/go-sasl"
|
||||||
"github.com/emersion/go-smtp"
|
"github.com/emersion/go-smtp"
|
||||||
|
"go.goblog.app/app/pkgs/bufferpool"
|
||||||
)
|
)
|
||||||
|
|
||||||
const defaultContactPath = "/contact"
|
const defaultContactPath = "/contact"
|
||||||
|
@ -30,7 +30,8 @@ func (a *goBlog) sendContactSubmission(w http.ResponseWriter, r *http.Request) {
|
||||||
// Get blog
|
// Get blog
|
||||||
_, bc := a.getBlog(r)
|
_, bc := a.getBlog(r)
|
||||||
// Get form values and build message
|
// Get form values and build message
|
||||||
var message bytes.Buffer
|
message := bufferpool.Get()
|
||||||
|
defer bufferpool.Put(message)
|
||||||
// Message
|
// Message
|
||||||
formMessage := cleanHTMLText(r.FormValue("message"))
|
formMessage := cleanHTMLText(r.FormValue("message"))
|
||||||
if formMessage == "" {
|
if formMessage == "" {
|
||||||
|
@ -39,20 +40,20 @@ func (a *goBlog) sendContactSubmission(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
// Name
|
// Name
|
||||||
if formName := cleanHTMLText(r.FormValue("name")); formName != "" {
|
if formName := cleanHTMLText(r.FormValue("name")); formName != "" {
|
||||||
_, _ = fmt.Fprintf(&message, "Name: %s\n", formName)
|
_, _ = fmt.Fprintf(message, "Name: %s\n", formName)
|
||||||
}
|
}
|
||||||
// Email
|
// Email
|
||||||
formEmail := cleanHTMLText(r.FormValue("email"))
|
formEmail := cleanHTMLText(r.FormValue("email"))
|
||||||
if formEmail != "" {
|
if formEmail != "" {
|
||||||
_, _ = fmt.Fprintf(&message, "Email: %s\n", formEmail)
|
_, _ = fmt.Fprintf(message, "Email: %s\n", formEmail)
|
||||||
}
|
}
|
||||||
// Website
|
// Website
|
||||||
if formWebsite := cleanHTMLText(r.FormValue("website")); formWebsite != "" {
|
if formWebsite := cleanHTMLText(r.FormValue("website")); formWebsite != "" {
|
||||||
_, _ = fmt.Fprintf(&message, "Website: %s\n", formWebsite)
|
_, _ = fmt.Fprintf(message, "Website: %s\n", formWebsite)
|
||||||
}
|
}
|
||||||
// Add line break if message is not empty
|
// Add line break if message is not empty
|
||||||
if message.Len() > 0 {
|
if message.Len() > 0 {
|
||||||
_, _ = fmt.Fprintf(&message, "\n")
|
_, _ = fmt.Fprintf(message, "\n")
|
||||||
}
|
}
|
||||||
// Add message text to message
|
// Add message text to message
|
||||||
_, _ = message.WriteString(formMessage)
|
_, _ = message.WriteString(formMessage)
|
||||||
|
@ -76,24 +77,25 @@ func (a *goBlog) sendContactEmail(cc *configContact, body, replyTo string) error
|
||||||
return fmt.Errorf("email not send as config is missing")
|
return fmt.Errorf("email not send as config is missing")
|
||||||
}
|
}
|
||||||
// Build email
|
// Build email
|
||||||
var email bytes.Buffer
|
email := bufferpool.Get()
|
||||||
_, _ = fmt.Fprintf(&email, "To: %s\n", cc.EmailTo)
|
defer bufferpool.Put(email)
|
||||||
|
_, _ = fmt.Fprintf(email, "To: %s\n", cc.EmailTo)
|
||||||
if replyTo != "" {
|
if replyTo != "" {
|
||||||
_, _ = fmt.Fprintf(&email, "Reply-To: %s\n", replyTo)
|
_, _ = fmt.Fprintf(email, "Reply-To: %s\n", replyTo)
|
||||||
}
|
}
|
||||||
_, _ = fmt.Fprintf(&email, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z))
|
_, _ = fmt.Fprintf(email, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z))
|
||||||
_, _ = fmt.Fprintf(&email, "From: %s\n", cc.EmailFrom)
|
_, _ = fmt.Fprintf(email, "From: %s\n", cc.EmailFrom)
|
||||||
subject := cc.EmailSubject
|
subject := cc.EmailSubject
|
||||||
if subject == "" {
|
if subject == "" {
|
||||||
subject = "New contact message"
|
subject = "New contact message"
|
||||||
}
|
}
|
||||||
_, _ = fmt.Fprintf(&email, "Subject: %s\n\n", subject)
|
_, _ = fmt.Fprintf(email, "Subject: %s\n\n", subject)
|
||||||
_, _ = fmt.Fprintf(&email, "%s\n", body)
|
_, _ = fmt.Fprintf(email, "%s\n", body)
|
||||||
// Send email using SMTP
|
// Send email using SMTP
|
||||||
auth := sasl.NewPlainClient("", cc.SMTPUser, cc.SMTPPassword)
|
auth := sasl.NewPlainClient("", cc.SMTPUser, cc.SMTPPassword)
|
||||||
port := cc.SMTPPort
|
port := cc.SMTPPort
|
||||||
if port == 0 {
|
if port == 0 {
|
||||||
port = 587
|
port = 587
|
||||||
}
|
}
|
||||||
return smtp.SendMail(cc.SMTPHost+":"+strconv.Itoa(port), auth, cc.EmailFrom, []string{cc.EmailTo}, bytes.NewReader(email.Bytes()))
|
return smtp.SendMail(cc.SMTPHost+":"+strconv.Itoa(port), auth, cc.EmailFrom, []string{cc.EmailTo}, email)
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,10 +5,10 @@ import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/cast"
|
"github.com/spf13/cast"
|
||||||
|
"go.goblog.app/app/pkgs/bufferpool"
|
||||||
)
|
)
|
||||||
|
|
||||||
const dbHooksBegin contextKey = "begin"
|
const dbHooksBegin contextKey = "begin"
|
||||||
|
@ -25,7 +25,7 @@ func (db *database) dbAfter(ctx context.Context, query string, args ...interface
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
dur := time.Since(ctx.Value(dbHooksBegin).(time.Time))
|
dur := time.Since(ctx.Value(dbHooksBegin).(time.Time))
|
||||||
var logBuilder strings.Builder
|
logBuilder := bufferpool.Get()
|
||||||
logBuilder.WriteString("\nQuery: ")
|
logBuilder.WriteString("\nQuery: ")
|
||||||
logBuilder.WriteString(`"`)
|
logBuilder.WriteString(`"`)
|
||||||
logBuilder.WriteString(query)
|
logBuilder.WriteString(query)
|
||||||
|
@ -52,6 +52,7 @@ func (db *database) dbAfter(ctx context.Context, query string, args ...interface
|
||||||
logBuilder.WriteString("\nDuration: ")
|
logBuilder.WriteString("\nDuration: ")
|
||||||
logBuilder.WriteString(dur.String())
|
logBuilder.WriteString(dur.String())
|
||||||
log.Println(logBuilder.String())
|
log.Println(logBuilder.String())
|
||||||
|
bufferpool.Put(logBuilder)
|
||||||
}
|
}
|
||||||
|
|
||||||
func argToString(arg interface{}) string {
|
func argToString(arg interface{}) string {
|
||||||
|
|
|
@ -11,6 +11,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"go.goblog.app/app/pkgs/bufferpool"
|
||||||
"go.goblog.app/app/pkgs/contenttype"
|
"go.goblog.app/app/pkgs/contenttype"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
ws "nhooyr.io/websocket"
|
ws "nhooyr.io/websocket"
|
||||||
|
@ -198,7 +199,8 @@ func (a *goBlog) editorPostTemplate(blog string, bc *configBlog) string {
|
||||||
|
|
||||||
func (a *goBlog) editorPostDesc(bc *configBlog) string {
|
func (a *goBlog) editorPostDesc(bc *configBlog) string {
|
||||||
t := a.ts.GetTemplateStringVariant(bc.Lang, "editorpostdesc")
|
t := a.ts.GetTemplateStringVariant(bc.Lang, "editorpostdesc")
|
||||||
var paramBuilder, statusBuilder strings.Builder
|
paramBuilder, statusBuilder := bufferpool.Get(), bufferpool.Get()
|
||||||
|
defer bufferpool.Put(paramBuilder, statusBuilder)
|
||||||
for i, param := range []string{
|
for i, param := range []string{
|
||||||
"published",
|
"published",
|
||||||
"updated",
|
"updated",
|
||||||
|
|
2
go.mod
2
go.mod
|
@ -32,7 +32,7 @@ require (
|
||||||
github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4
|
github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4
|
||||||
github.com/justinas/alice v1.2.0
|
github.com/justinas/alice v1.2.0
|
||||||
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9
|
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9
|
||||||
github.com/klauspost/compress v1.14.3
|
github.com/klauspost/compress v1.14.4
|
||||||
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
|
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
|
||||||
github.com/lopezator/migrator v0.3.0
|
github.com/lopezator/migrator v0.3.0
|
||||||
github.com/mattn/go-sqlite3 v1.14.11
|
github.com/mattn/go-sqlite3 v1.14.11
|
||||||
|
|
4
go.sum
4
go.sum
|
@ -275,8 +275,8 @@ github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9 h1:+9REu9CK9D1AQ
|
||||||
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9/go.mod h1:OvY5ZBrAC9kOvM2PZs9Lw0BH+5K7tjrT6T7SFhn27OA=
|
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9/go.mod h1:OvY5ZBrAC9kOvM2PZs9Lw0BH+5K7tjrT6T7SFhn27OA=
|
||||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||||
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||||
github.com/klauspost/compress v1.14.3 h1:DQv1WP+iS4srNjibdnHtqu8JNWCDMluj5NzPnFJsnvk=
|
github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4=
|
||||||
github.com/klauspost/compress v1.14.3/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||||
|
|
11
hooks.go
11
hooks.go
|
@ -1,11 +1,12 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"html/template"
|
"html/template"
|
||||||
"log"
|
"log"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"go.goblog.app/app/pkgs/bufferpool"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (a *goBlog) preStartHooks() {
|
func (a *goBlog) preStartHooks() {
|
||||||
|
@ -91,13 +92,13 @@ func (cfg *configHooks) executeTemplateCommand(hookType string, tmpl string, dat
|
||||||
log.Println("Failed to parse cmd template:", err.Error())
|
log.Println("Failed to parse cmd template:", err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var cmdBuf bytes.Buffer
|
cmdBuf := bufferpool.Get()
|
||||||
if err = cmdTmpl.Execute(&cmdBuf, data); err != nil {
|
defer bufferpool.Put(cmdBuf)
|
||||||
|
if err = cmdTmpl.Execute(cmdBuf, data); err != nil {
|
||||||
log.Println("Failed to execute cmd template:", err.Error())
|
log.Println("Failed to execute cmd template:", err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
cmd := cmdBuf.String()
|
executeHookCommand(hookType, cfg.Shell, cmdBuf.String())
|
||||||
executeHookCommand(hookType, cfg.Shell, cmd)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type hourlyHookFunc func()
|
type hourlyHookFunc func()
|
||||||
|
|
|
@ -4,6 +4,8 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"go.goblog.app/app/pkgs/bufferpool"
|
||||||
)
|
)
|
||||||
|
|
||||||
func noIndexHeader(next http.Handler) http.Handler {
|
func noIndexHeader(next http.Handler) http.Handler {
|
||||||
|
@ -37,7 +39,7 @@ func headAsGetHandler(next http.Handler) http.Handler {
|
||||||
|
|
||||||
func (a *goBlog) securityHeaders(next http.Handler) http.Handler {
|
func (a *goBlog) securityHeaders(next http.Handler) http.Handler {
|
||||||
// Build CSP domains list
|
// Build CSP domains list
|
||||||
var cspBuilder strings.Builder
|
cspBuilder := bufferpool.Get()
|
||||||
if mp := a.cfg.Micropub.MediaStorage; mp != nil && mp.MediaURL != "" {
|
if mp := a.cfg.Micropub.MediaStorage; mp != nil && mp.MediaURL != "" {
|
||||||
if u, err := url.Parse(mp.MediaURL); err == nil {
|
if u, err := url.Parse(mp.MediaURL); err == nil {
|
||||||
cspBuilder.WriteByte(' ')
|
cspBuilder.WriteByte(' ')
|
||||||
|
@ -49,6 +51,7 @@ func (a *goBlog) securityHeaders(next http.Handler) http.Handler {
|
||||||
cspBuilder.WriteString(strings.Join(a.cfg.Server.CSPDomains, " "))
|
cspBuilder.WriteString(strings.Join(a.cfg.Server.CSPDomains, " "))
|
||||||
}
|
}
|
||||||
cspDomains := cspBuilder.String()
|
cspDomains := cspBuilder.String()
|
||||||
|
bufferpool.Put(cspBuilder)
|
||||||
// Return handler
|
// Return handler
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
w.Header().Set("Strict-Transport-Security", "max-age=31536000;")
|
w.Header().Set("Strict-Transport-Security", "max-age=31536000;")
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"errors"
|
"errors"
|
||||||
|
@ -62,7 +61,8 @@ func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc *http
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
// Compress
|
// Compress
|
||||||
var imgBuffer bytes.Buffer
|
imgBuffer := bufferpool.Get()
|
||||||
|
defer bufferpool.Put(imgBuffer)
|
||||||
err := requests.
|
err := requests.
|
||||||
URL("https://api.shortpixel.com/v2/reducer-sync.php").
|
URL("https://api.shortpixel.com/v2/reducer-sync.php").
|
||||||
Client(hc).
|
Client(hc).
|
||||||
|
@ -78,14 +78,14 @@ func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc *http
|
||||||
"keep_exif": 0,
|
"keep_exif": 0,
|
||||||
"url": url,
|
"url": url,
|
||||||
}).
|
}).
|
||||||
ToBytesBuffer(&imgBuffer).
|
ToBytesBuffer(imgBuffer).
|
||||||
Fetch(context.Background())
|
Fetch(context.Background())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println("Shortpixel error:", err.Error())
|
log.Println("Shortpixel error:", err.Error())
|
||||||
return "", errors.New("failed to compress image using shortpixel")
|
return "", errors.New("failed to compress image using shortpixel")
|
||||||
}
|
}
|
||||||
// Upload compressed file
|
// Upload compressed file
|
||||||
return uploadCompressedFile(fileExtension, &imgBuffer, upload)
|
return uploadCompressedFile(fileExtension, imgBuffer, upload)
|
||||||
}
|
}
|
||||||
|
|
||||||
type tinify struct {
|
type tinify struct {
|
||||||
|
@ -123,7 +123,8 @@ func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Cli
|
||||||
return "", tinifyErr
|
return "", tinifyErr
|
||||||
}
|
}
|
||||||
// Resize and download image
|
// Resize and download image
|
||||||
var imgBuffer bytes.Buffer
|
imgBuffer := bufferpool.Get()
|
||||||
|
defer bufferpool.Put(imgBuffer)
|
||||||
err = requests.
|
err = requests.
|
||||||
URL(compressedLocation).
|
URL(compressedLocation).
|
||||||
Client(hc).
|
Client(hc).
|
||||||
|
@ -136,14 +137,14 @@ func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Cli
|
||||||
"height": defaultCompressionHeight,
|
"height": defaultCompressionHeight,
|
||||||
},
|
},
|
||||||
}).
|
}).
|
||||||
ToBytesBuffer(&imgBuffer).
|
ToBytesBuffer(imgBuffer).
|
||||||
Fetch(context.Background())
|
Fetch(context.Background())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println("Tinify error:", err.Error())
|
log.Println("Tinify error:", err.Error())
|
||||||
return "", tinifyErr
|
return "", tinifyErr
|
||||||
}
|
}
|
||||||
// Upload compressed file
|
// Upload compressed file
|
||||||
return uploadCompressedFile(fileExtension, &imgBuffer, upload)
|
return uploadCompressedFile(fileExtension, imgBuffer, upload)
|
||||||
}
|
}
|
||||||
|
|
||||||
type cloudflare struct{}
|
type cloudflare struct{}
|
||||||
|
@ -156,18 +157,19 @@ func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http
|
||||||
// Force jpeg
|
// Force jpeg
|
||||||
fileExtension := "jpeg"
|
fileExtension := "jpeg"
|
||||||
// Compress
|
// Compress
|
||||||
var imgBuffer bytes.Buffer
|
imgBuffer := bufferpool.Get()
|
||||||
|
defer bufferpool.Put(imgBuffer)
|
||||||
err := requests.
|
err := requests.
|
||||||
URL(fmt.Sprintf("https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=%d,h=%d/%s", defaultCompressionWidth, defaultCompressionHeight, url)).
|
URL(fmt.Sprintf("https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=%d,h=%d/%s", defaultCompressionWidth, defaultCompressionHeight, url)).
|
||||||
Client(hc).
|
Client(hc).
|
||||||
ToBytesBuffer(&imgBuffer).
|
ToBytesBuffer(imgBuffer).
|
||||||
Fetch(context.Background())
|
Fetch(context.Background())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println("Cloudflare error:", err.Error())
|
log.Println("Cloudflare error:", err.Error())
|
||||||
return "", errors.New("failed to compress image using cloudflare")
|
return "", errors.New("failed to compress image using cloudflare")
|
||||||
}
|
}
|
||||||
// Upload compressed file
|
// Upload compressed file
|
||||||
return uploadCompressedFile(fileExtension, &imgBuffer, upload)
|
return uploadCompressedFile(fileExtension, imgBuffer, upload)
|
||||||
}
|
}
|
||||||
|
|
||||||
func uploadCompressedFile(fileExtension string, r io.Reader, upload mediaStorageSaveFunc) (string, error) {
|
func uploadCompressedFile(fileExtension string, r io.Reader, upload mediaStorageSaveFunc) (string, error) {
|
||||||
|
|
16
posts.go
16
posts.go
|
@ -12,6 +12,7 @@ import (
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/vcraescu/go-paginator"
|
"github.com/vcraescu/go-paginator"
|
||||||
|
"go.goblog.app/app/pkgs/bufferpool"
|
||||||
)
|
)
|
||||||
|
|
||||||
var errPostNotFound = errors.New("post not found")
|
var errPostNotFound = errors.New("post not found")
|
||||||
|
@ -213,24 +214,24 @@ func (a *goBlog) serveDate(w http.ResponseWriter, r *http.Request) {
|
||||||
a.serve404(w, r)
|
a.serve404(w, r)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var title, dPath strings.Builder
|
title, dPath := bufferpool.Get(), bufferpool.Get()
|
||||||
if year != 0 {
|
if year != 0 {
|
||||||
_, _ = fmt.Fprintf(&title, "%0004d", year)
|
_, _ = fmt.Fprintf(title, "%0004d", year)
|
||||||
_, _ = fmt.Fprintf(&dPath, "%0004d", year)
|
_, _ = fmt.Fprintf(dPath, "%0004d", year)
|
||||||
} else {
|
} else {
|
||||||
_, _ = title.WriteString("XXXX")
|
_, _ = title.WriteString("XXXX")
|
||||||
_, _ = dPath.WriteString("x")
|
_, _ = dPath.WriteString("x")
|
||||||
}
|
}
|
||||||
if month != 0 {
|
if month != 0 {
|
||||||
_, _ = fmt.Fprintf(&title, "-%02d", month)
|
_, _ = fmt.Fprintf(title, "-%02d", month)
|
||||||
_, _ = fmt.Fprintf(&dPath, "/%02d", month)
|
_, _ = fmt.Fprintf(dPath, "/%02d", month)
|
||||||
} else if day != 0 {
|
} else if day != 0 {
|
||||||
_, _ = title.WriteString("-XX")
|
_, _ = title.WriteString("-XX")
|
||||||
_, _ = dPath.WriteString("/x")
|
_, _ = dPath.WriteString("/x")
|
||||||
}
|
}
|
||||||
if day != 0 {
|
if day != 0 {
|
||||||
_, _ = fmt.Fprintf(&title, "-%02d", day)
|
_, _ = fmt.Fprintf(title, "-%02d", day)
|
||||||
_, _ = fmt.Fprintf(&dPath, "/%02d", day)
|
_, _ = fmt.Fprintf(dPath, "/%02d", day)
|
||||||
}
|
}
|
||||||
_, bc := a.getBlog(r)
|
_, bc := a.getBlog(r)
|
||||||
a.serveIndex(w, r.WithContext(context.WithValue(r.Context(), indexConfigKey, &indexConfig{
|
a.serveIndex(w, r.WithContext(context.WithValue(r.Context(), indexConfigKey, &indexConfig{
|
||||||
|
@ -240,6 +241,7 @@ func (a *goBlog) serveDate(w http.ResponseWriter, r *http.Request) {
|
||||||
day: day,
|
day: day,
|
||||||
title: title.String(),
|
title: title.String(),
|
||||||
})))
|
})))
|
||||||
|
bufferpool.Put(title, dPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
type indexConfig struct {
|
type indexConfig struct {
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
|
|
||||||
"github.com/araddon/dateparse"
|
"github.com/araddon/dateparse"
|
||||||
"github.com/thoas/go-funk"
|
"github.com/thoas/go-funk"
|
||||||
|
"go.goblog.app/app/pkgs/bufferpool"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (a *goBlog) checkPost(p *post) (err error) {
|
func (a *goBlog) checkPost(p *post) (err error) {
|
||||||
|
@ -630,7 +631,7 @@ group by name;
|
||||||
|
|
||||||
func (db *database) usesOfMediaFile(names ...string) (counts []int, err error) {
|
func (db *database) usesOfMediaFile(names ...string) (counts []int, err error) {
|
||||||
sqlArgs := []interface{}{dbNoCache}
|
sqlArgs := []interface{}{dbNoCache}
|
||||||
var nameValues strings.Builder
|
nameValues := bufferpool.Get()
|
||||||
for i, n := range names {
|
for i, n := range names {
|
||||||
if i > 0 {
|
if i > 0 {
|
||||||
nameValues.WriteString(", ")
|
nameValues.WriteString(", ")
|
||||||
|
@ -642,6 +643,7 @@ func (db *database) usesOfMediaFile(names ...string) (counts []int, err error) {
|
||||||
sqlArgs = append(sqlArgs, sql.Named(named, n))
|
sqlArgs = append(sqlArgs, sql.Named(named, n))
|
||||||
}
|
}
|
||||||
rows, err := db.query(fmt.Sprintf(mediaUseSql, nameValues.String()), sqlArgs...)
|
rows, err := db.query(fmt.Sprintf(mediaUseSql, nameValues.String()), sqlArgs...)
|
||||||
|
bufferpool.Put(nameValues)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
4
tts.go
4
tts.go
|
@ -17,6 +17,7 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/carlmjohnson/requests"
|
"github.com/carlmjohnson/requests"
|
||||||
|
"go.goblog.app/app/pkgs/bufferpool"
|
||||||
"go.goblog.app/app/pkgs/mp3merge"
|
"go.goblog.app/app/pkgs/mp3merge"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -109,7 +110,8 @@ func (a *goBlog) createPostTTSAudio(p *post) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Merge partsBuffers into final buffer
|
// Merge partsBuffers into final buffer
|
||||||
final := new(bytes.Buffer)
|
final := bufferpool.Get()
|
||||||
|
defer bufferpool.Put(final)
|
||||||
hash := sha256.New()
|
hash := sha256.New()
|
||||||
if err := mp3merge.MergeMP3(io.MultiWriter(final, hash), partsBuffers...); err != nil {
|
if err := mp3merge.MergeMP3(io.MultiWriter(final, hash), partsBuffers...); err != nil {
|
||||||
return err
|
return err
|
||||||
|
|
10
ui_test.go
10
ui_test.go
|
@ -10,6 +10,7 @@ import (
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"go.goblog.app/app/pkgs/bufferpool"
|
||||||
)
|
)
|
||||||
|
|
||||||
var _ io.Writer = &htmlBuilder{}
|
var _ io.Writer = &htmlBuilder{}
|
||||||
|
@ -30,16 +31,17 @@ func Test_renderPostTax(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
buf := &bytes.Buffer{}
|
buf := bufferpool.Get()
|
||||||
|
defer bufferpool.Put(buf)
|
||||||
|
|
||||||
hb := newHtmlBuilder(buf)
|
hb := newHtmlBuilder(buf)
|
||||||
|
|
||||||
app.renderPostTax(hb, p, app.cfg.Blogs["default"])
|
app.renderPostTax(hb, p, app.cfg.Blogs["default"])
|
||||||
res := buf.String()
|
|
||||||
|
|
||||||
_, err := goquery.NewDocumentFromReader(strings.NewReader(res))
|
_, err := goquery.NewDocumentFromReader(strings.NewReader(buf.String()))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
assert.Equal(t, "<p><strong>Tags</strong>: <a class=\"p-category\" rel=\"tag\" href=\"/tags/bar\">Bar</a>, <a class=\"p-category\" rel=\"tag\" href=\"/tags/foo\">Foo</a></p>", res)
|
assert.Equal(t, "<p><strong>Tags</strong>: <a class=\"p-category\" rel=\"tag\" href=\"/tags/bar\">Bar</a>, <a class=\"p-category\" rel=\"tag\" href=\"/tags/foo\">Foo</a></p>", buf.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
func Test_renderOldContentWarning(t *testing.T) {
|
func Test_renderOldContentWarning(t *testing.T) {
|
||||||
|
|
|
@ -11,42 +11,51 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
"github.com/thoas/go-funk"
|
"github.com/thoas/go-funk"
|
||||||
|
"go.goblog.app/app/pkgs/bufferpool"
|
||||||
"go.goblog.app/app/pkgs/contenttype"
|
"go.goblog.app/app/pkgs/contenttype"
|
||||||
"willnorris.com/go/microformats"
|
"willnorris.com/go/microformats"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (a *goBlog) initWebmentionQueue() {
|
func (a *goBlog) initWebmentionQueue() {
|
||||||
go func() {
|
go func() {
|
||||||
for {
|
done := false
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
|
a.shutdown.Add(func() {
|
||||||
|
done = true
|
||||||
|
wg.Wait()
|
||||||
|
log.Println("Stopped webmention queue")
|
||||||
|
})
|
||||||
|
for !done {
|
||||||
qi, err := a.db.peekQueue("wm")
|
qi, err := a.db.peekQueue("wm")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println("webmention queue:", err.Error())
|
log.Println("webmention queue:", err.Error())
|
||||||
continue
|
continue
|
||||||
} else if qi != nil {
|
}
|
||||||
var m mention
|
if qi == nil {
|
||||||
err = gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&m)
|
|
||||||
if err != nil {
|
|
||||||
log.Println("webmention queue:", err.Error())
|
|
||||||
_ = a.db.dequeue(qi)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
err = a.verifyMention(&m)
|
|
||||||
if err != nil {
|
|
||||||
log.Println(fmt.Sprintf("Failed to verify webmention from %s to %s: %s", m.Source, m.Target, err.Error()))
|
|
||||||
}
|
|
||||||
err = a.db.dequeue(qi)
|
|
||||||
if err != nil {
|
|
||||||
log.Println("webmention queue:", err.Error())
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// No item in the queue, wait a moment
|
// No item in the queue, wait a moment
|
||||||
time.Sleep(15 * time.Second)
|
time.Sleep(5 * time.Second)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var m mention
|
||||||
|
if err = gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&m); err != nil {
|
||||||
|
log.Println("webmention queue:", err.Error())
|
||||||
|
_ = a.db.dequeue(qi)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err = a.verifyMention(&m); err != nil {
|
||||||
|
log.Println(fmt.Sprintf("Failed to verify webmention from %s to %s: %s", m.Source, m.Target, err.Error()))
|
||||||
|
}
|
||||||
|
if err = a.db.dequeue(qi); err != nil {
|
||||||
|
log.Println("webmention queue:", err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
wg.Done()
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -54,8 +63,9 @@ func (a *goBlog) queueMention(m *mention) error {
|
||||||
if wm := a.cfg.Webmention; wm != nil && wm.DisableReceiving {
|
if wm := a.cfg.Webmention; wm != nil && wm.DisableReceiving {
|
||||||
return errors.New("webmention receiving disabled")
|
return errors.New("webmention receiving disabled")
|
||||||
}
|
}
|
||||||
var buf bytes.Buffer
|
buf := bufferpool.Get()
|
||||||
if err := gob.NewEncoder(&buf).Encode(m); err != nil {
|
defer bufferpool.Put(buf)
|
||||||
|
if err := gob.NewEncoder(buf).Encode(m); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return a.db.enqueue("wm", buf.Bytes(), time.Now())
|
return a.db.enqueue("wm", buf.Bytes(), time.Now())
|
||||||
|
@ -165,12 +175,13 @@ func (a *goBlog) verifyMention(m *mention) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *goBlog) verifyReader(m *mention, body io.Reader) error {
|
func (a *goBlog) verifyReader(m *mention, body io.Reader) error {
|
||||||
var linksBuffer, gqBuffer, mfBuffer bytes.Buffer
|
linksBuffer, gqBuffer, mfBuffer := bufferpool.Get(), bufferpool.Get(), bufferpool.Get()
|
||||||
if _, err := io.Copy(io.MultiWriter(&linksBuffer, &gqBuffer, &mfBuffer), body); err != nil {
|
defer bufferpool.Put(linksBuffer, gqBuffer, mfBuffer)
|
||||||
|
if _, err := io.Copy(io.MultiWriter(linksBuffer, gqBuffer, mfBuffer), body); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// Check if source mentions target
|
// Check if source mentions target
|
||||||
links, err := allLinksFromHTML(&linksBuffer, defaultIfEmpty(m.NewSource, m.Source))
|
links, err := allLinksFromHTML(linksBuffer, defaultIfEmpty(m.NewSource, m.Source))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -210,13 +221,13 @@ func (a *goBlog) verifyReader(m *mention, body io.Reader) error {
|
||||||
m.Author = ""
|
m.Author = ""
|
||||||
m.Url = ""
|
m.Url = ""
|
||||||
m.hasUrl = false
|
m.hasUrl = false
|
||||||
m.fillFromData(microformats.Parse(&mfBuffer, sourceURL))
|
m.fillFromData(microformats.Parse(mfBuffer, sourceURL))
|
||||||
if m.Url == "" {
|
if m.Url == "" {
|
||||||
m.Url = m.Source
|
m.Url = m.Source
|
||||||
}
|
}
|
||||||
// Set title when content is empty as well
|
// Set title when content is empty as well
|
||||||
if m.Title == "" && m.Content == "" {
|
if m.Title == "" && m.Content == "" {
|
||||||
doc, err := goquery.NewDocumentFromReader(&gqBuffer)
|
doc, err := goquery.NewDocumentFromReader(gqBuffer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue