More pooled buffers and don't AP announce replies

This commit is contained in:
Jan-Lukas Else 2022-02-22 16:52:03 +01:00
parent b453e6b400
commit 0f1408fe3e
17 changed files with 174 additions and 150 deletions

View File

@ -49,7 +49,9 @@ func (a *goBlog) initActivityPub() error {
a.apDelete(p)
})
a.pUndeleteHooks = append(a.pUndeleteHooks, func(p *post) {
a.apUndelete(p)
if p.isPublishedSectionPost() {
a.apUndelete(p)
}
})
// Prepare webfinger
a.webfingerResources = map[string]*configBlog{}
@ -185,7 +187,7 @@ func (a *goBlog) apHandleInbox(w http.ResponseWriter, r *http.Request) {
// May be a mention; find links to blog and save them as webmentions
if links, err := allLinksFromHTMLString(content, baseUrl); err == nil {
for _, link := range links {
if strings.HasPrefix(link, blogIri) {
if strings.HasPrefix(link, a.cfg.Server.PublicAddress) {
_ = a.createWebmention(baseUrl, link)
}
}
@ -313,11 +315,6 @@ func (a *goBlog) apPost(p *post) {
"type": "Create",
"object": n,
})
if n.InReplyTo != "" {
// Is reply, so announce it
time.Sleep(30 * time.Second)
a.apAnnounce(p)
}
}
func (a *goBlog) apUpdate(p *post) {
@ -331,17 +328,6 @@ func (a *goBlog) apUpdate(p *post) {
})
}
func (a *goBlog) apAnnounce(p *post) {
a.apSendToAllFollowers(p.Blog, map[string]interface{}{
"@context": []string{asContext},
"actor": a.apIri(a.cfg.Blogs[p.Blog]),
"id": a.fullPostURL(p) + "#announce",
"published": a.toASNote(p).Published,
"type": "Announce",
"object": a.fullPostURL(p),
})
}
func (a *goBlog) apDelete(p *post) {
a.apSendToAllFollowers(p.Blog, map[string]interface{}{
"@context": []string{asContext},
@ -450,17 +436,16 @@ func (a *goBlog) loadActivityPubPrivateKey() error {
}
}
// Generate and cache key
key, err := rsa.GenerateKey(rand.Reader, 2048)
var err error
a.apPrivateKey, err = rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return err
}
encodedKey := x509.MarshalPKCS1PrivateKey(key)
pemEncoded := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: encodedKey})
err = a.db.cachePersistently("activitypub_key", pemEncoded)
if err != nil {
return err
}
a.apPrivateKey = key
// Return key
return nil
return a.db.cachePersistently(
"activitypub_key",
pem.EncodeToMemory(&pem.Block{
Type: "PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(a.apPrivateKey),
}),
)
}

View File

@ -6,11 +6,14 @@ import (
"encoding/gob"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/url"
"sync"
"time"
"go.goblog.app/app/pkgs/bufferpool"
"go.goblog.app/app/pkgs/contenttype"
)
@ -22,40 +25,49 @@ type apRequest struct {
func (a *goBlog) initAPSendQueue() {
go func() {
for {
done := false
var wg sync.WaitGroup
wg.Add(1)
a.shutdown.Add(func() {
done = true
wg.Wait()
log.Println("Stopped AP send queue")
})
for !done {
qi, err := a.db.peekQueue("ap")
if err != nil {
log.Println("activitypub send queue:", err.Error())
continue
} else if qi != nil {
var r apRequest
err = gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&r)
if err != nil {
log.Println("activitypub send queue:", err.Error())
_ = a.db.dequeue(qi)
}
if qi == nil {
// No item in the queue, wait a moment
time.Sleep(5 * time.Second)
continue
}
var r apRequest
if err = gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&r); err != nil {
log.Println("activitypub send queue:", err.Error())
_ = a.db.dequeue(qi)
continue
}
if err = a.apSendSigned(r.BlogIri, r.To, r.Activity); err != nil {
if r.Try++; r.Try < 20 {
// Try it again
buf := bufferpool.Get()
_ = r.encode(buf)
qi.content = buf.Bytes()
_ = a.db.reschedule(qi, time.Duration(r.Try)*10*time.Minute)
bufferpool.Put(buf)
continue
}
if err := a.apSendSigned(r.BlogIri, r.To, r.Activity); err != nil {
if r.Try++; r.Try < 20 {
// Try it again
qi.content, _ = r.encode()
_ = a.db.reschedule(qi, time.Duration(r.Try)*10*time.Minute)
continue
} else {
log.Printf("Request to %s failed for the 20th time", r.To)
log.Println()
_ = a.db.apRemoveInbox(r.To)
}
}
err = a.db.dequeue(qi)
if err != nil {
log.Println("activitypub send queue:", err.Error())
}
} else {
// No item in the queue, wait a moment
time.Sleep(15 * time.Second)
log.Println("AP request failed for the 20th time:", r.To)
_ = a.db.apRemoveInbox(r.To)
}
if err = a.db.dequeue(qi); err != nil {
log.Println("activitypub send queue:", err.Error())
}
}
wg.Done()
}()
}
@ -64,24 +76,20 @@ func (db *database) apQueueSendSigned(blogIri, to string, activity interface{})
if err != nil {
return err
}
b, err := (&apRequest{
buf := bufferpool.Get()
defer bufferpool.Put(buf)
if err := (&apRequest{
BlogIri: blogIri,
To: to,
Activity: body,
}).encode()
if err != nil {
}).encode(buf); err != nil {
return err
}
return db.enqueue("ap", b, time.Now())
return db.enqueue("ap", buf.Bytes(), time.Now())
}
func (r *apRequest) encode() ([]byte, error) {
var buf bytes.Buffer
err := gob.NewEncoder(&buf).Encode(r)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
func (r *apRequest) encode(w io.Writer) error {
return gob.NewEncoder(w).Encode(r)
}
func (a *goBlog) apSendSigned(blogIri, to string, activity []byte) error {
@ -89,9 +97,7 @@ func (a *goBlog) apSendSigned(blogIri, to string, activity []byte) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
// Create request
var requestBuffer bytes.Buffer
requestBuffer.Write(activity)
r, err := http.NewRequestWithContext(ctx, http.MethodPost, to, &requestBuffer)
r, err := http.NewRequestWithContext(ctx, http.MethodPost, to, bytes.NewReader(activity))
if err != nil {
return err
}
@ -117,7 +123,7 @@ func (a *goBlog) apSendSigned(blogIri, to string, activity []byte) error {
if err != nil {
return err
}
defer resp.Body.Close()
_ = resp.Body.Close()
if !apRequestIsSuccess(resp.StatusCode) {
return fmt.Errorf("signed request failed with status %d", resp.StatusCode)
}

View File

@ -3,7 +3,6 @@ package main
import (
"bytes"
"context"
"io"
"log"
"net/http"
"sort"
@ -67,7 +66,7 @@ func (a *goBlog) serveBlogrollExport(w http.ResponseWriter, r *http.Request) {
return
}
w.Header().Set(contentType, contenttype.XMLUTF8)
_, _ = io.Copy(w, opmlBuf)
_, _ = opmlBuf.WriteTo(w)
}
func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) {
@ -108,13 +107,14 @@ func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) {
}
func (db *database) cacheOutlines(blog string, outlines []*opml.Outline) {
var opmlBuffer bytes.Buffer
_ = opml.Render(&opmlBuffer, &opml.OPML{
opmlBuffer := bufferpool.Get()
_ = opml.Render(opmlBuffer, &opml.OPML{
Version: "2.0",
DateCreated: time.Now().UTC(),
Outlines: outlines,
})
_ = db.cachePersistently("blogroll_"+blog, opmlBuffer.Bytes())
bufferpool.Put(opmlBuffer)
}
func (db *database) loadOutlineCache(blog string) []*opml.Outline {

View File

@ -5,6 +5,8 @@ import (
"database/sql"
"encoding/gob"
"net/http"
"go.goblog.app/app/pkgs/bufferpool"
)
const (
@ -195,9 +197,10 @@ func (db *database) getBlogStats(blog string) (data *blogStatsData, err error) {
}
func (db *database) cacheBlogStats(blog string, stats *blogStatsData) {
var buf bytes.Buffer
_ = gob.NewEncoder(&buf).Encode(stats)
buf := bufferpool.Get()
_ = gob.NewEncoder(buf).Encode(stats)
_ = db.cachePersistently("blogstats_"+blog, buf.Bytes())
bufferpool.Put(buf)
}
func (db *database) loadBlogStatsCache(blog string) (stats *blogStatsData) {

View File

@ -1,7 +1,6 @@
package main
import (
"bytes"
"fmt"
"log"
"net/http"
@ -10,6 +9,7 @@ import (
"github.com/emersion/go-sasl"
"github.com/emersion/go-smtp"
"go.goblog.app/app/pkgs/bufferpool"
)
const defaultContactPath = "/contact"
@ -30,7 +30,8 @@ func (a *goBlog) sendContactSubmission(w http.ResponseWriter, r *http.Request) {
// Get blog
_, bc := a.getBlog(r)
// Get form values and build message
var message bytes.Buffer
message := bufferpool.Get()
defer bufferpool.Put(message)
// Message
formMessage := cleanHTMLText(r.FormValue("message"))
if formMessage == "" {
@ -39,20 +40,20 @@ func (a *goBlog) sendContactSubmission(w http.ResponseWriter, r *http.Request) {
}
// Name
if formName := cleanHTMLText(r.FormValue("name")); formName != "" {
_, _ = fmt.Fprintf(&message, "Name: %s\n", formName)
_, _ = fmt.Fprintf(message, "Name: %s\n", formName)
}
// Email
formEmail := cleanHTMLText(r.FormValue("email"))
if formEmail != "" {
_, _ = fmt.Fprintf(&message, "Email: %s\n", formEmail)
_, _ = fmt.Fprintf(message, "Email: %s\n", formEmail)
}
// Website
if formWebsite := cleanHTMLText(r.FormValue("website")); formWebsite != "" {
_, _ = fmt.Fprintf(&message, "Website: %s\n", formWebsite)
_, _ = fmt.Fprintf(message, "Website: %s\n", formWebsite)
}
// Add line break if message is not empty
if message.Len() > 0 {
_, _ = fmt.Fprintf(&message, "\n")
_, _ = fmt.Fprintf(message, "\n")
}
// Add message text to message
_, _ = message.WriteString(formMessage)
@ -76,24 +77,25 @@ func (a *goBlog) sendContactEmail(cc *configContact, body, replyTo string) error
return fmt.Errorf("email not send as config is missing")
}
// Build email
var email bytes.Buffer
_, _ = fmt.Fprintf(&email, "To: %s\n", cc.EmailTo)
email := bufferpool.Get()
defer bufferpool.Put(email)
_, _ = fmt.Fprintf(email, "To: %s\n", cc.EmailTo)
if replyTo != "" {
_, _ = fmt.Fprintf(&email, "Reply-To: %s\n", replyTo)
_, _ = fmt.Fprintf(email, "Reply-To: %s\n", replyTo)
}
_, _ = fmt.Fprintf(&email, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z))
_, _ = fmt.Fprintf(&email, "From: %s\n", cc.EmailFrom)
_, _ = fmt.Fprintf(email, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z))
_, _ = fmt.Fprintf(email, "From: %s\n", cc.EmailFrom)
subject := cc.EmailSubject
if subject == "" {
subject = "New contact message"
}
_, _ = fmt.Fprintf(&email, "Subject: %s\n\n", subject)
_, _ = fmt.Fprintf(&email, "%s\n", body)
_, _ = fmt.Fprintf(email, "Subject: %s\n\n", subject)
_, _ = fmt.Fprintf(email, "%s\n", body)
// Send email using SMTP
auth := sasl.NewPlainClient("", cc.SMTPUser, cc.SMTPPassword)
port := cc.SMTPPort
if port == 0 {
port = 587
}
return smtp.SendMail(cc.SMTPHost+":"+strconv.Itoa(port), auth, cc.EmailFrom, []string{cc.EmailTo}, bytes.NewReader(email.Bytes()))
return smtp.SendMail(cc.SMTPHost+":"+strconv.Itoa(port), auth, cc.EmailFrom, []string{cc.EmailTo}, email)
}

View File

@ -5,10 +5,10 @@ import (
"database/sql"
"fmt"
"log"
"strings"
"time"
"github.com/spf13/cast"
"go.goblog.app/app/pkgs/bufferpool"
)
const dbHooksBegin contextKey = "begin"
@ -25,7 +25,7 @@ func (db *database) dbAfter(ctx context.Context, query string, args ...interface
return
}
dur := time.Since(ctx.Value(dbHooksBegin).(time.Time))
var logBuilder strings.Builder
logBuilder := bufferpool.Get()
logBuilder.WriteString("\nQuery: ")
logBuilder.WriteString(`"`)
logBuilder.WriteString(query)
@ -52,6 +52,7 @@ func (db *database) dbAfter(ctx context.Context, query string, args ...interface
logBuilder.WriteString("\nDuration: ")
logBuilder.WriteString(dur.String())
log.Println(logBuilder.String())
bufferpool.Put(logBuilder)
}
func argToString(arg interface{}) string {

View File

@ -11,6 +11,7 @@ import (
"strings"
"time"
"go.goblog.app/app/pkgs/bufferpool"
"go.goblog.app/app/pkgs/contenttype"
"gopkg.in/yaml.v3"
ws "nhooyr.io/websocket"
@ -198,7 +199,8 @@ func (a *goBlog) editorPostTemplate(blog string, bc *configBlog) string {
func (a *goBlog) editorPostDesc(bc *configBlog) string {
t := a.ts.GetTemplateStringVariant(bc.Lang, "editorpostdesc")
var paramBuilder, statusBuilder strings.Builder
paramBuilder, statusBuilder := bufferpool.Get(), bufferpool.Get()
defer bufferpool.Put(paramBuilder, statusBuilder)
for i, param := range []string{
"published",
"updated",

2
go.mod
View File

@ -32,7 +32,7 @@ require (
github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4
github.com/justinas/alice v1.2.0
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9
github.com/klauspost/compress v1.14.3
github.com/klauspost/compress v1.14.4
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
github.com/lopezator/migrator v0.3.0
github.com/mattn/go-sqlite3 v1.14.11

4
go.sum
View File

@ -275,8 +275,8 @@ github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9 h1:+9REu9CK9D1AQ
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9/go.mod h1:OvY5ZBrAC9kOvM2PZs9Lw0BH+5K7tjrT6T7SFhn27OA=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.14.3 h1:DQv1WP+iS4srNjibdnHtqu8JNWCDMluj5NzPnFJsnvk=
github.com/klauspost/compress v1.14.3/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4=
github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=

View File

@ -1,11 +1,12 @@
package main
import (
"bytes"
"html/template"
"log"
"os/exec"
"time"
"go.goblog.app/app/pkgs/bufferpool"
)
func (a *goBlog) preStartHooks() {
@ -91,13 +92,13 @@ func (cfg *configHooks) executeTemplateCommand(hookType string, tmpl string, dat
log.Println("Failed to parse cmd template:", err.Error())
return
}
var cmdBuf bytes.Buffer
if err = cmdTmpl.Execute(&cmdBuf, data); err != nil {
cmdBuf := bufferpool.Get()
defer bufferpool.Put(cmdBuf)
if err = cmdTmpl.Execute(cmdBuf, data); err != nil {
log.Println("Failed to execute cmd template:", err.Error())
return
}
cmd := cmdBuf.String()
executeHookCommand(hookType, cfg.Shell, cmd)
executeHookCommand(hookType, cfg.Shell, cmdBuf.String())
}
type hourlyHookFunc func()

View File

@ -4,6 +4,8 @@ import (
"net/http"
"net/url"
"strings"
"go.goblog.app/app/pkgs/bufferpool"
)
func noIndexHeader(next http.Handler) http.Handler {
@ -37,7 +39,7 @@ func headAsGetHandler(next http.Handler) http.Handler {
func (a *goBlog) securityHeaders(next http.Handler) http.Handler {
// Build CSP domains list
var cspBuilder strings.Builder
cspBuilder := bufferpool.Get()
if mp := a.cfg.Micropub.MediaStorage; mp != nil && mp.MediaURL != "" {
if u, err := url.Parse(mp.MediaURL); err == nil {
cspBuilder.WriteByte(' ')
@ -49,6 +51,7 @@ func (a *goBlog) securityHeaders(next http.Handler) http.Handler {
cspBuilder.WriteString(strings.Join(a.cfg.Server.CSPDomains, " "))
}
cspDomains := cspBuilder.String()
bufferpool.Put(cspBuilder)
// Return handler
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Strict-Transport-Security", "max-age=31536000;")

View File

@ -1,7 +1,6 @@
package main
import (
"bytes"
"context"
"crypto/sha256"
"errors"
@ -62,7 +61,8 @@ func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc *http
return "", nil
}
// Compress
var imgBuffer bytes.Buffer
imgBuffer := bufferpool.Get()
defer bufferpool.Put(imgBuffer)
err := requests.
URL("https://api.shortpixel.com/v2/reducer-sync.php").
Client(hc).
@ -78,14 +78,14 @@ func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc *http
"keep_exif": 0,
"url": url,
}).
ToBytesBuffer(&imgBuffer).
ToBytesBuffer(imgBuffer).
Fetch(context.Background())
if err != nil {
log.Println("Shortpixel error:", err.Error())
return "", errors.New("failed to compress image using shortpixel")
}
// Upload compressed file
return uploadCompressedFile(fileExtension, &imgBuffer, upload)
return uploadCompressedFile(fileExtension, imgBuffer, upload)
}
type tinify struct {
@ -123,7 +123,8 @@ func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Cli
return "", tinifyErr
}
// Resize and download image
var imgBuffer bytes.Buffer
imgBuffer := bufferpool.Get()
defer bufferpool.Put(imgBuffer)
err = requests.
URL(compressedLocation).
Client(hc).
@ -136,14 +137,14 @@ func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Cli
"height": defaultCompressionHeight,
},
}).
ToBytesBuffer(&imgBuffer).
ToBytesBuffer(imgBuffer).
Fetch(context.Background())
if err != nil {
log.Println("Tinify error:", err.Error())
return "", tinifyErr
}
// Upload compressed file
return uploadCompressedFile(fileExtension, &imgBuffer, upload)
return uploadCompressedFile(fileExtension, imgBuffer, upload)
}
type cloudflare struct{}
@ -156,18 +157,19 @@ func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http
// Force jpeg
fileExtension := "jpeg"
// Compress
var imgBuffer bytes.Buffer
imgBuffer := bufferpool.Get()
defer bufferpool.Put(imgBuffer)
err := requests.
URL(fmt.Sprintf("https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=%d,h=%d/%s", defaultCompressionWidth, defaultCompressionHeight, url)).
Client(hc).
ToBytesBuffer(&imgBuffer).
ToBytesBuffer(imgBuffer).
Fetch(context.Background())
if err != nil {
log.Println("Cloudflare error:", err.Error())
return "", errors.New("failed to compress image using cloudflare")
}
// Upload compressed file
return uploadCompressedFile(fileExtension, &imgBuffer, upload)
return uploadCompressedFile(fileExtension, imgBuffer, upload)
}
func uploadCompressedFile(fileExtension string, r io.Reader, upload mediaStorageSaveFunc) (string, error) {

View File

@ -12,6 +12,7 @@ import (
"github.com/go-chi/chi/v5"
"github.com/vcraescu/go-paginator"
"go.goblog.app/app/pkgs/bufferpool"
)
var errPostNotFound = errors.New("post not found")
@ -213,24 +214,24 @@ func (a *goBlog) serveDate(w http.ResponseWriter, r *http.Request) {
a.serve404(w, r)
return
}
var title, dPath strings.Builder
title, dPath := bufferpool.Get(), bufferpool.Get()
if year != 0 {
_, _ = fmt.Fprintf(&title, "%0004d", year)
_, _ = fmt.Fprintf(&dPath, "%0004d", year)
_, _ = fmt.Fprintf(title, "%0004d", year)
_, _ = fmt.Fprintf(dPath, "%0004d", year)
} else {
_, _ = title.WriteString("XXXX")
_, _ = dPath.WriteString("x")
}
if month != 0 {
_, _ = fmt.Fprintf(&title, "-%02d", month)
_, _ = fmt.Fprintf(&dPath, "/%02d", month)
_, _ = fmt.Fprintf(title, "-%02d", month)
_, _ = fmt.Fprintf(dPath, "/%02d", month)
} else if day != 0 {
_, _ = title.WriteString("-XX")
_, _ = dPath.WriteString("/x")
}
if day != 0 {
_, _ = fmt.Fprintf(&title, "-%02d", day)
_, _ = fmt.Fprintf(&dPath, "/%02d", day)
_, _ = fmt.Fprintf(title, "-%02d", day)
_, _ = fmt.Fprintf(dPath, "/%02d", day)
}
_, bc := a.getBlog(r)
a.serveIndex(w, r.WithContext(context.WithValue(r.Context(), indexConfigKey, &indexConfig{
@ -240,6 +241,7 @@ func (a *goBlog) serveDate(w http.ResponseWriter, r *http.Request) {
day: day,
title: title.String(),
})))
bufferpool.Put(title, dPath)
}
type indexConfig struct {

View File

@ -12,6 +12,7 @@ import (
"github.com/araddon/dateparse"
"github.com/thoas/go-funk"
"go.goblog.app/app/pkgs/bufferpool"
)
func (a *goBlog) checkPost(p *post) (err error) {
@ -630,7 +631,7 @@ group by name;
func (db *database) usesOfMediaFile(names ...string) (counts []int, err error) {
sqlArgs := []interface{}{dbNoCache}
var nameValues strings.Builder
nameValues := bufferpool.Get()
for i, n := range names {
if i > 0 {
nameValues.WriteString(", ")
@ -642,6 +643,7 @@ func (db *database) usesOfMediaFile(names ...string) (counts []int, err error) {
sqlArgs = append(sqlArgs, sql.Named(named, n))
}
rows, err := db.query(fmt.Sprintf(mediaUseSql, nameValues.String()), sqlArgs...)
bufferpool.Put(nameValues)
if err != nil {
return nil, err
}

4
tts.go
View File

@ -17,6 +17,7 @@ import (
"sync"
"github.com/carlmjohnson/requests"
"go.goblog.app/app/pkgs/bufferpool"
"go.goblog.app/app/pkgs/mp3merge"
)
@ -109,7 +110,8 @@ func (a *goBlog) createPostTTSAudio(p *post) error {
}
// Merge partsBuffers into final buffer
final := new(bytes.Buffer)
final := bufferpool.Get()
defer bufferpool.Put(final)
hash := sha256.New()
if err := mp3merge.MergeMP3(io.MultiWriter(final, hash), partsBuffers...); err != nil {
return err

View File

@ -10,6 +10,7 @@ import (
"github.com/PuerkitoBio/goquery"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.goblog.app/app/pkgs/bufferpool"
)
var _ io.Writer = &htmlBuilder{}
@ -30,16 +31,17 @@ func Test_renderPostTax(t *testing.T) {
},
}
buf := &bytes.Buffer{}
buf := bufferpool.Get()
defer bufferpool.Put(buf)
hb := newHtmlBuilder(buf)
app.renderPostTax(hb, p, app.cfg.Blogs["default"])
res := buf.String()
_, err := goquery.NewDocumentFromReader(strings.NewReader(res))
_, err := goquery.NewDocumentFromReader(strings.NewReader(buf.String()))
require.NoError(t, err)
assert.Equal(t, "<p><strong>Tags</strong>: <a class=\"p-category\" rel=\"tag\" href=\"/tags/bar\">Bar</a>, <a class=\"p-category\" rel=\"tag\" href=\"/tags/foo\">Foo</a></p>", res)
assert.Equal(t, "<p><strong>Tags</strong>: <a class=\"p-category\" rel=\"tag\" href=\"/tags/bar\">Bar</a>, <a class=\"p-category\" rel=\"tag\" href=\"/tags/foo\">Foo</a></p>", buf.String())
}
func Test_renderOldContentWarning(t *testing.T) {

View File

@ -11,42 +11,51 @@ import (
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/thoas/go-funk"
"go.goblog.app/app/pkgs/bufferpool"
"go.goblog.app/app/pkgs/contenttype"
"willnorris.com/go/microformats"
)
func (a *goBlog) initWebmentionQueue() {
go func() {
for {
done := false
var wg sync.WaitGroup
wg.Add(1)
a.shutdown.Add(func() {
done = true
wg.Wait()
log.Println("Stopped webmention queue")
})
for !done {
qi, err := a.db.peekQueue("wm")
if err != nil {
log.Println("webmention queue:", err.Error())
continue
} else if qi != nil {
var m mention
err = gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&m)
if err != nil {
log.Println("webmention queue:", err.Error())
_ = a.db.dequeue(qi)
continue
}
err = a.verifyMention(&m)
if err != nil {
log.Println(fmt.Sprintf("Failed to verify webmention from %s to %s: %s", m.Source, m.Target, err.Error()))
}
err = a.db.dequeue(qi)
if err != nil {
log.Println("webmention queue:", err.Error())
}
} else {
}
if qi == nil {
// No item in the queue, wait a moment
time.Sleep(15 * time.Second)
time.Sleep(5 * time.Second)
continue
}
var m mention
if err = gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&m); err != nil {
log.Println("webmention queue:", err.Error())
_ = a.db.dequeue(qi)
continue
}
if err = a.verifyMention(&m); err != nil {
log.Println(fmt.Sprintf("Failed to verify webmention from %s to %s: %s", m.Source, m.Target, err.Error()))
}
if err = a.db.dequeue(qi); err != nil {
log.Println("webmention queue:", err.Error())
}
}
wg.Done()
}()
}
@ -54,8 +63,9 @@ func (a *goBlog) queueMention(m *mention) error {
if wm := a.cfg.Webmention; wm != nil && wm.DisableReceiving {
return errors.New("webmention receiving disabled")
}
var buf bytes.Buffer
if err := gob.NewEncoder(&buf).Encode(m); err != nil {
buf := bufferpool.Get()
defer bufferpool.Put(buf)
if err := gob.NewEncoder(buf).Encode(m); err != nil {
return err
}
return a.db.enqueue("wm", buf.Bytes(), time.Now())
@ -165,12 +175,13 @@ func (a *goBlog) verifyMention(m *mention) error {
}
func (a *goBlog) verifyReader(m *mention, body io.Reader) error {
var linksBuffer, gqBuffer, mfBuffer bytes.Buffer
if _, err := io.Copy(io.MultiWriter(&linksBuffer, &gqBuffer, &mfBuffer), body); err != nil {
linksBuffer, gqBuffer, mfBuffer := bufferpool.Get(), bufferpool.Get(), bufferpool.Get()
defer bufferpool.Put(linksBuffer, gqBuffer, mfBuffer)
if _, err := io.Copy(io.MultiWriter(linksBuffer, gqBuffer, mfBuffer), body); err != nil {
return err
}
// Check if source mentions target
links, err := allLinksFromHTML(&linksBuffer, defaultIfEmpty(m.NewSource, m.Source))
links, err := allLinksFromHTML(linksBuffer, defaultIfEmpty(m.NewSource, m.Source))
if err != nil {
return err
}
@ -210,13 +221,13 @@ func (a *goBlog) verifyReader(m *mention, body io.Reader) error {
m.Author = ""
m.Url = ""
m.hasUrl = false
m.fillFromData(microformats.Parse(&mfBuffer, sourceURL))
m.fillFromData(microformats.Parse(mfBuffer, sourceURL))
if m.Url == "" {
m.Url = m.Source
}
// Set title when content is empty as well
if m.Title == "" && m.Content == "" {
doc, err := goquery.NewDocumentFromReader(&gqBuffer)
doc, err := goquery.NewDocumentFromReader(gqBuffer)
if err != nil {
return err
}