From 0f1408fe3e58a1721b4d2eb2e29cdc9b3dd6f3de Mon Sep 17 00:00:00 2001 From: Jan-Lukas Else Date: Tue, 22 Feb 2022 16:52:03 +0100 Subject: [PATCH] More pooled buffers and don't AP announce replies --- activityPub.go | 41 ++++++------------ activityPubSending.go | 88 +++++++++++++++++++++------------------ blogroll.go | 8 ++-- blogstats.go | 7 +++- contact.go | 30 ++++++------- databaseHooks.go | 5 ++- editor.go | 4 +- go.mod | 2 +- go.sum | 4 +- hooks.go | 11 ++--- httpMiddlewares.go | 5 ++- mediaCompression.go | 22 +++++----- posts.go | 16 +++---- postsDb.go | 4 +- tts.go | 4 +- ui_test.go | 10 +++-- webmentionVerification.go | 63 ++++++++++++++++------------ 17 files changed, 174 insertions(+), 150 deletions(-) diff --git a/activityPub.go b/activityPub.go index 7a0ea86..8be529a 100644 --- a/activityPub.go +++ b/activityPub.go @@ -49,7 +49,9 @@ func (a *goBlog) initActivityPub() error { a.apDelete(p) }) a.pUndeleteHooks = append(a.pUndeleteHooks, func(p *post) { - a.apUndelete(p) + if p.isPublishedSectionPost() { + a.apUndelete(p) + } }) // Prepare webfinger a.webfingerResources = map[string]*configBlog{} @@ -185,7 +187,7 @@ func (a *goBlog) apHandleInbox(w http.ResponseWriter, r *http.Request) { // May be a mention; find links to blog and save them as webmentions if links, err := allLinksFromHTMLString(content, baseUrl); err == nil { for _, link := range links { - if strings.HasPrefix(link, blogIri) { + if strings.HasPrefix(link, a.cfg.Server.PublicAddress) { _ = a.createWebmention(baseUrl, link) } } @@ -313,11 +315,6 @@ func (a *goBlog) apPost(p *post) { "type": "Create", "object": n, }) - if n.InReplyTo != "" { - // Is reply, so announce it - time.Sleep(30 * time.Second) - a.apAnnounce(p) - } } func (a *goBlog) apUpdate(p *post) { @@ -331,17 +328,6 @@ func (a *goBlog) apUpdate(p *post) { }) } -func (a *goBlog) apAnnounce(p *post) { - a.apSendToAllFollowers(p.Blog, map[string]interface{}{ - "@context": []string{asContext}, - "actor": a.apIri(a.cfg.Blogs[p.Blog]), - "id": a.fullPostURL(p) + "#announce", - "published": a.toASNote(p).Published, - "type": "Announce", - "object": a.fullPostURL(p), - }) -} - func (a *goBlog) apDelete(p *post) { a.apSendToAllFollowers(p.Blog, map[string]interface{}{ "@context": []string{asContext}, @@ -450,17 +436,16 @@ func (a *goBlog) loadActivityPubPrivateKey() error { } } // Generate and cache key - key, err := rsa.GenerateKey(rand.Reader, 2048) + var err error + a.apPrivateKey, err = rsa.GenerateKey(rand.Reader, 2048) if err != nil { return err } - encodedKey := x509.MarshalPKCS1PrivateKey(key) - pemEncoded := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: encodedKey}) - err = a.db.cachePersistently("activitypub_key", pemEncoded) - if err != nil { - return err - } - a.apPrivateKey = key - // Return key - return nil + return a.db.cachePersistently( + "activitypub_key", + pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(a.apPrivateKey), + }), + ) } diff --git a/activityPubSending.go b/activityPubSending.go index 411cdc0..3d3288f 100644 --- a/activityPubSending.go +++ b/activityPubSending.go @@ -6,11 +6,14 @@ import ( "encoding/gob" "encoding/json" "fmt" + "io" "log" "net/http" "net/url" + "sync" "time" + "go.goblog.app/app/pkgs/bufferpool" "go.goblog.app/app/pkgs/contenttype" ) @@ -22,40 +25,49 @@ type apRequest struct { func (a *goBlog) initAPSendQueue() { go func() { - for { + done := false + var wg sync.WaitGroup + wg.Add(1) + a.shutdown.Add(func() { + done = true + wg.Wait() + log.Println("Stopped AP send queue") + }) + for !done { qi, err := a.db.peekQueue("ap") if err != nil { log.Println("activitypub send queue:", err.Error()) continue - } else if qi != nil { - var r apRequest - err = gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&r) - if err != nil { - log.Println("activitypub send queue:", err.Error()) - _ = a.db.dequeue(qi) + } + if qi == nil { + // No item in the queue, wait a moment + time.Sleep(5 * time.Second) + continue + } + var r apRequest + if err = gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&r); err != nil { + log.Println("activitypub send queue:", err.Error()) + _ = a.db.dequeue(qi) + continue + } + if err = a.apSendSigned(r.BlogIri, r.To, r.Activity); err != nil { + if r.Try++; r.Try < 20 { + // Try it again + buf := bufferpool.Get() + _ = r.encode(buf) + qi.content = buf.Bytes() + _ = a.db.reschedule(qi, time.Duration(r.Try)*10*time.Minute) + bufferpool.Put(buf) continue } - if err := a.apSendSigned(r.BlogIri, r.To, r.Activity); err != nil { - if r.Try++; r.Try < 20 { - // Try it again - qi.content, _ = r.encode() - _ = a.db.reschedule(qi, time.Duration(r.Try)*10*time.Minute) - continue - } else { - log.Printf("Request to %s failed for the 20th time", r.To) - log.Println() - _ = a.db.apRemoveInbox(r.To) - } - } - err = a.db.dequeue(qi) - if err != nil { - log.Println("activitypub send queue:", err.Error()) - } - } else { - // No item in the queue, wait a moment - time.Sleep(15 * time.Second) + log.Println("AP request failed for the 20th time:", r.To) + _ = a.db.apRemoveInbox(r.To) + } + if err = a.db.dequeue(qi); err != nil { + log.Println("activitypub send queue:", err.Error()) } } + wg.Done() }() } @@ -64,24 +76,20 @@ func (db *database) apQueueSendSigned(blogIri, to string, activity interface{}) if err != nil { return err } - b, err := (&apRequest{ + buf := bufferpool.Get() + defer bufferpool.Put(buf) + if err := (&apRequest{ BlogIri: blogIri, To: to, Activity: body, - }).encode() - if err != nil { + }).encode(buf); err != nil { return err } - return db.enqueue("ap", b, time.Now()) + return db.enqueue("ap", buf.Bytes(), time.Now()) } -func (r *apRequest) encode() ([]byte, error) { - var buf bytes.Buffer - err := gob.NewEncoder(&buf).Encode(r) - if err != nil { - return nil, err - } - return buf.Bytes(), nil +func (r *apRequest) encode(w io.Writer) error { + return gob.NewEncoder(w).Encode(r) } func (a *goBlog) apSendSigned(blogIri, to string, activity []byte) error { @@ -89,9 +97,7 @@ func (a *goBlog) apSendSigned(blogIri, to string, activity []byte) error { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() // Create request - var requestBuffer bytes.Buffer - requestBuffer.Write(activity) - r, err := http.NewRequestWithContext(ctx, http.MethodPost, to, &requestBuffer) + r, err := http.NewRequestWithContext(ctx, http.MethodPost, to, bytes.NewReader(activity)) if err != nil { return err } @@ -117,7 +123,7 @@ func (a *goBlog) apSendSigned(blogIri, to string, activity []byte) error { if err != nil { return err } - defer resp.Body.Close() + _ = resp.Body.Close() if !apRequestIsSuccess(resp.StatusCode) { return fmt.Errorf("signed request failed with status %d", resp.StatusCode) } diff --git a/blogroll.go b/blogroll.go index 26203a7..83d908b 100644 --- a/blogroll.go +++ b/blogroll.go @@ -3,7 +3,6 @@ package main import ( "bytes" "context" - "io" "log" "net/http" "sort" @@ -67,7 +66,7 @@ func (a *goBlog) serveBlogrollExport(w http.ResponseWriter, r *http.Request) { return } w.Header().Set(contentType, contenttype.XMLUTF8) - _, _ = io.Copy(w, opmlBuf) + _, _ = opmlBuf.WriteTo(w) } func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) { @@ -108,13 +107,14 @@ func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) { } func (db *database) cacheOutlines(blog string, outlines []*opml.Outline) { - var opmlBuffer bytes.Buffer - _ = opml.Render(&opmlBuffer, &opml.OPML{ + opmlBuffer := bufferpool.Get() + _ = opml.Render(opmlBuffer, &opml.OPML{ Version: "2.0", DateCreated: time.Now().UTC(), Outlines: outlines, }) _ = db.cachePersistently("blogroll_"+blog, opmlBuffer.Bytes()) + bufferpool.Put(opmlBuffer) } func (db *database) loadOutlineCache(blog string) []*opml.Outline { diff --git a/blogstats.go b/blogstats.go index e243be1..6d9329f 100644 --- a/blogstats.go +++ b/blogstats.go @@ -5,6 +5,8 @@ import ( "database/sql" "encoding/gob" "net/http" + + "go.goblog.app/app/pkgs/bufferpool" ) const ( @@ -195,9 +197,10 @@ func (db *database) getBlogStats(blog string) (data *blogStatsData, err error) { } func (db *database) cacheBlogStats(blog string, stats *blogStatsData) { - var buf bytes.Buffer - _ = gob.NewEncoder(&buf).Encode(stats) + buf := bufferpool.Get() + _ = gob.NewEncoder(buf).Encode(stats) _ = db.cachePersistently("blogstats_"+blog, buf.Bytes()) + bufferpool.Put(buf) } func (db *database) loadBlogStatsCache(blog string) (stats *blogStatsData) { diff --git a/contact.go b/contact.go index 63637c4..b94974d 100644 --- a/contact.go +++ b/contact.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "fmt" "log" "net/http" @@ -10,6 +9,7 @@ import ( "github.com/emersion/go-sasl" "github.com/emersion/go-smtp" + "go.goblog.app/app/pkgs/bufferpool" ) const defaultContactPath = "/contact" @@ -30,7 +30,8 @@ func (a *goBlog) sendContactSubmission(w http.ResponseWriter, r *http.Request) { // Get blog _, bc := a.getBlog(r) // Get form values and build message - var message bytes.Buffer + message := bufferpool.Get() + defer bufferpool.Put(message) // Message formMessage := cleanHTMLText(r.FormValue("message")) if formMessage == "" { @@ -39,20 +40,20 @@ func (a *goBlog) sendContactSubmission(w http.ResponseWriter, r *http.Request) { } // Name if formName := cleanHTMLText(r.FormValue("name")); formName != "" { - _, _ = fmt.Fprintf(&message, "Name: %s\n", formName) + _, _ = fmt.Fprintf(message, "Name: %s\n", formName) } // Email formEmail := cleanHTMLText(r.FormValue("email")) if formEmail != "" { - _, _ = fmt.Fprintf(&message, "Email: %s\n", formEmail) + _, _ = fmt.Fprintf(message, "Email: %s\n", formEmail) } // Website if formWebsite := cleanHTMLText(r.FormValue("website")); formWebsite != "" { - _, _ = fmt.Fprintf(&message, "Website: %s\n", formWebsite) + _, _ = fmt.Fprintf(message, "Website: %s\n", formWebsite) } // Add line break if message is not empty if message.Len() > 0 { - _, _ = fmt.Fprintf(&message, "\n") + _, _ = fmt.Fprintf(message, "\n") } // Add message text to message _, _ = message.WriteString(formMessage) @@ -76,24 +77,25 @@ func (a *goBlog) sendContactEmail(cc *configContact, body, replyTo string) error return fmt.Errorf("email not send as config is missing") } // Build email - var email bytes.Buffer - _, _ = fmt.Fprintf(&email, "To: %s\n", cc.EmailTo) + email := bufferpool.Get() + defer bufferpool.Put(email) + _, _ = fmt.Fprintf(email, "To: %s\n", cc.EmailTo) if replyTo != "" { - _, _ = fmt.Fprintf(&email, "Reply-To: %s\n", replyTo) + _, _ = fmt.Fprintf(email, "Reply-To: %s\n", replyTo) } - _, _ = fmt.Fprintf(&email, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z)) - _, _ = fmt.Fprintf(&email, "From: %s\n", cc.EmailFrom) + _, _ = fmt.Fprintf(email, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z)) + _, _ = fmt.Fprintf(email, "From: %s\n", cc.EmailFrom) subject := cc.EmailSubject if subject == "" { subject = "New contact message" } - _, _ = fmt.Fprintf(&email, "Subject: %s\n\n", subject) - _, _ = fmt.Fprintf(&email, "%s\n", body) + _, _ = fmt.Fprintf(email, "Subject: %s\n\n", subject) + _, _ = fmt.Fprintf(email, "%s\n", body) // Send email using SMTP auth := sasl.NewPlainClient("", cc.SMTPUser, cc.SMTPPassword) port := cc.SMTPPort if port == 0 { port = 587 } - return smtp.SendMail(cc.SMTPHost+":"+strconv.Itoa(port), auth, cc.EmailFrom, []string{cc.EmailTo}, bytes.NewReader(email.Bytes())) + return smtp.SendMail(cc.SMTPHost+":"+strconv.Itoa(port), auth, cc.EmailFrom, []string{cc.EmailTo}, email) } diff --git a/databaseHooks.go b/databaseHooks.go index 0b24d2f..68ae3d9 100644 --- a/databaseHooks.go +++ b/databaseHooks.go @@ -5,10 +5,10 @@ import ( "database/sql" "fmt" "log" - "strings" "time" "github.com/spf13/cast" + "go.goblog.app/app/pkgs/bufferpool" ) const dbHooksBegin contextKey = "begin" @@ -25,7 +25,7 @@ func (db *database) dbAfter(ctx context.Context, query string, args ...interface return } dur := time.Since(ctx.Value(dbHooksBegin).(time.Time)) - var logBuilder strings.Builder + logBuilder := bufferpool.Get() logBuilder.WriteString("\nQuery: ") logBuilder.WriteString(`"`) logBuilder.WriteString(query) @@ -52,6 +52,7 @@ func (db *database) dbAfter(ctx context.Context, query string, args ...interface logBuilder.WriteString("\nDuration: ") logBuilder.WriteString(dur.String()) log.Println(logBuilder.String()) + bufferpool.Put(logBuilder) } func argToString(arg interface{}) string { diff --git a/editor.go b/editor.go index 9f50165..f053f5f 100644 --- a/editor.go +++ b/editor.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "go.goblog.app/app/pkgs/bufferpool" "go.goblog.app/app/pkgs/contenttype" "gopkg.in/yaml.v3" ws "nhooyr.io/websocket" @@ -198,7 +199,8 @@ func (a *goBlog) editorPostTemplate(blog string, bc *configBlog) string { func (a *goBlog) editorPostDesc(bc *configBlog) string { t := a.ts.GetTemplateStringVariant(bc.Lang, "editorpostdesc") - var paramBuilder, statusBuilder strings.Builder + paramBuilder, statusBuilder := bufferpool.Get(), bufferpool.Get() + defer bufferpool.Put(paramBuilder, statusBuilder) for i, param := range []string{ "published", "updated", diff --git a/go.mod b/go.mod index 0259e27..c4af781 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4 github.com/justinas/alice v1.2.0 github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9 - github.com/klauspost/compress v1.14.3 + github.com/klauspost/compress v1.14.4 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible github.com/lopezator/migrator v0.3.0 github.com/mattn/go-sqlite3 v1.14.11 diff --git a/go.sum b/go.sum index a97ea19..c0f7065 100644 --- a/go.sum +++ b/go.sum @@ -275,8 +275,8 @@ github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9 h1:+9REu9CK9D1AQ github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9/go.mod h1:OvY5ZBrAC9kOvM2PZs9Lw0BH+5K7tjrT6T7SFhn27OA= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.14.3 h1:DQv1WP+iS4srNjibdnHtqu8JNWCDMluj5NzPnFJsnvk= -github.com/klauspost/compress v1.14.3/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4= +github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= diff --git a/hooks.go b/hooks.go index 8e901a2..78537eb 100644 --- a/hooks.go +++ b/hooks.go @@ -1,11 +1,12 @@ package main import ( - "bytes" "html/template" "log" "os/exec" "time" + + "go.goblog.app/app/pkgs/bufferpool" ) func (a *goBlog) preStartHooks() { @@ -91,13 +92,13 @@ func (cfg *configHooks) executeTemplateCommand(hookType string, tmpl string, dat log.Println("Failed to parse cmd template:", err.Error()) return } - var cmdBuf bytes.Buffer - if err = cmdTmpl.Execute(&cmdBuf, data); err != nil { + cmdBuf := bufferpool.Get() + defer bufferpool.Put(cmdBuf) + if err = cmdTmpl.Execute(cmdBuf, data); err != nil { log.Println("Failed to execute cmd template:", err.Error()) return } - cmd := cmdBuf.String() - executeHookCommand(hookType, cfg.Shell, cmd) + executeHookCommand(hookType, cfg.Shell, cmdBuf.String()) } type hourlyHookFunc func() diff --git a/httpMiddlewares.go b/httpMiddlewares.go index 2702a6b..d15256a 100644 --- a/httpMiddlewares.go +++ b/httpMiddlewares.go @@ -4,6 +4,8 @@ import ( "net/http" "net/url" "strings" + + "go.goblog.app/app/pkgs/bufferpool" ) func noIndexHeader(next http.Handler) http.Handler { @@ -37,7 +39,7 @@ func headAsGetHandler(next http.Handler) http.Handler { func (a *goBlog) securityHeaders(next http.Handler) http.Handler { // Build CSP domains list - var cspBuilder strings.Builder + cspBuilder := bufferpool.Get() if mp := a.cfg.Micropub.MediaStorage; mp != nil && mp.MediaURL != "" { if u, err := url.Parse(mp.MediaURL); err == nil { cspBuilder.WriteByte(' ') @@ -49,6 +51,7 @@ func (a *goBlog) securityHeaders(next http.Handler) http.Handler { cspBuilder.WriteString(strings.Join(a.cfg.Server.CSPDomains, " ")) } cspDomains := cspBuilder.String() + bufferpool.Put(cspBuilder) // Return handler return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Strict-Transport-Security", "max-age=31536000;") diff --git a/mediaCompression.go b/mediaCompression.go index e9ae634..932b4d5 100644 --- a/mediaCompression.go +++ b/mediaCompression.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "context" "crypto/sha256" "errors" @@ -62,7 +61,8 @@ func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc *http return "", nil } // Compress - var imgBuffer bytes.Buffer + imgBuffer := bufferpool.Get() + defer bufferpool.Put(imgBuffer) err := requests. URL("https://api.shortpixel.com/v2/reducer-sync.php"). Client(hc). @@ -78,14 +78,14 @@ func (sp *shortpixel) compress(url string, upload mediaStorageSaveFunc, hc *http "keep_exif": 0, "url": url, }). - ToBytesBuffer(&imgBuffer). + ToBytesBuffer(imgBuffer). Fetch(context.Background()) if err != nil { log.Println("Shortpixel error:", err.Error()) return "", errors.New("failed to compress image using shortpixel") } // Upload compressed file - return uploadCompressedFile(fileExtension, &imgBuffer, upload) + return uploadCompressedFile(fileExtension, imgBuffer, upload) } type tinify struct { @@ -123,7 +123,8 @@ func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Cli return "", tinifyErr } // Resize and download image - var imgBuffer bytes.Buffer + imgBuffer := bufferpool.Get() + defer bufferpool.Put(imgBuffer) err = requests. URL(compressedLocation). Client(hc). @@ -136,14 +137,14 @@ func (tf *tinify) compress(url string, upload mediaStorageSaveFunc, hc *http.Cli "height": defaultCompressionHeight, }, }). - ToBytesBuffer(&imgBuffer). + ToBytesBuffer(imgBuffer). Fetch(context.Background()) if err != nil { log.Println("Tinify error:", err.Error()) return "", tinifyErr } // Upload compressed file - return uploadCompressedFile(fileExtension, &imgBuffer, upload) + return uploadCompressedFile(fileExtension, imgBuffer, upload) } type cloudflare struct{} @@ -156,18 +157,19 @@ func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http // Force jpeg fileExtension := "jpeg" // Compress - var imgBuffer bytes.Buffer + imgBuffer := bufferpool.Get() + defer bufferpool.Put(imgBuffer) err := requests. URL(fmt.Sprintf("https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=%d,h=%d/%s", defaultCompressionWidth, defaultCompressionHeight, url)). Client(hc). - ToBytesBuffer(&imgBuffer). + ToBytesBuffer(imgBuffer). Fetch(context.Background()) if err != nil { log.Println("Cloudflare error:", err.Error()) return "", errors.New("failed to compress image using cloudflare") } // Upload compressed file - return uploadCompressedFile(fileExtension, &imgBuffer, upload) + return uploadCompressedFile(fileExtension, imgBuffer, upload) } func uploadCompressedFile(fileExtension string, r io.Reader, upload mediaStorageSaveFunc) (string, error) { diff --git a/posts.go b/posts.go index 4af2723..f89e0c4 100644 --- a/posts.go +++ b/posts.go @@ -12,6 +12,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/vcraescu/go-paginator" + "go.goblog.app/app/pkgs/bufferpool" ) var errPostNotFound = errors.New("post not found") @@ -213,24 +214,24 @@ func (a *goBlog) serveDate(w http.ResponseWriter, r *http.Request) { a.serve404(w, r) return } - var title, dPath strings.Builder + title, dPath := bufferpool.Get(), bufferpool.Get() if year != 0 { - _, _ = fmt.Fprintf(&title, "%0004d", year) - _, _ = fmt.Fprintf(&dPath, "%0004d", year) + _, _ = fmt.Fprintf(title, "%0004d", year) + _, _ = fmt.Fprintf(dPath, "%0004d", year) } else { _, _ = title.WriteString("XXXX") _, _ = dPath.WriteString("x") } if month != 0 { - _, _ = fmt.Fprintf(&title, "-%02d", month) - _, _ = fmt.Fprintf(&dPath, "/%02d", month) + _, _ = fmt.Fprintf(title, "-%02d", month) + _, _ = fmt.Fprintf(dPath, "/%02d", month) } else if day != 0 { _, _ = title.WriteString("-XX") _, _ = dPath.WriteString("/x") } if day != 0 { - _, _ = fmt.Fprintf(&title, "-%02d", day) - _, _ = fmt.Fprintf(&dPath, "/%02d", day) + _, _ = fmt.Fprintf(title, "-%02d", day) + _, _ = fmt.Fprintf(dPath, "/%02d", day) } _, bc := a.getBlog(r) a.serveIndex(w, r.WithContext(context.WithValue(r.Context(), indexConfigKey, &indexConfig{ @@ -240,6 +241,7 @@ func (a *goBlog) serveDate(w http.ResponseWriter, r *http.Request) { day: day, title: title.String(), }))) + bufferpool.Put(title, dPath) } type indexConfig struct { diff --git a/postsDb.go b/postsDb.go index 02d137e..17f0434 100644 --- a/postsDb.go +++ b/postsDb.go @@ -12,6 +12,7 @@ import ( "github.com/araddon/dateparse" "github.com/thoas/go-funk" + "go.goblog.app/app/pkgs/bufferpool" ) func (a *goBlog) checkPost(p *post) (err error) { @@ -630,7 +631,7 @@ group by name; func (db *database) usesOfMediaFile(names ...string) (counts []int, err error) { sqlArgs := []interface{}{dbNoCache} - var nameValues strings.Builder + nameValues := bufferpool.Get() for i, n := range names { if i > 0 { nameValues.WriteString(", ") @@ -642,6 +643,7 @@ func (db *database) usesOfMediaFile(names ...string) (counts []int, err error) { sqlArgs = append(sqlArgs, sql.Named(named, n)) } rows, err := db.query(fmt.Sprintf(mediaUseSql, nameValues.String()), sqlArgs...) + bufferpool.Put(nameValues) if err != nil { return nil, err } diff --git a/tts.go b/tts.go index 16e2cdc..c67e839 100644 --- a/tts.go +++ b/tts.go @@ -17,6 +17,7 @@ import ( "sync" "github.com/carlmjohnson/requests" + "go.goblog.app/app/pkgs/bufferpool" "go.goblog.app/app/pkgs/mp3merge" ) @@ -109,7 +110,8 @@ func (a *goBlog) createPostTTSAudio(p *post) error { } // Merge partsBuffers into final buffer - final := new(bytes.Buffer) + final := bufferpool.Get() + defer bufferpool.Put(final) hash := sha256.New() if err := mp3merge.MergeMP3(io.MultiWriter(final, hash), partsBuffers...); err != nil { return err diff --git a/ui_test.go b/ui_test.go index 99d9746..1464ce5 100644 --- a/ui_test.go +++ b/ui_test.go @@ -10,6 +10,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.goblog.app/app/pkgs/bufferpool" ) var _ io.Writer = &htmlBuilder{} @@ -30,16 +31,17 @@ func Test_renderPostTax(t *testing.T) { }, } - buf := &bytes.Buffer{} + buf := bufferpool.Get() + defer bufferpool.Put(buf) + hb := newHtmlBuilder(buf) app.renderPostTax(hb, p, app.cfg.Blogs["default"]) - res := buf.String() - _, err := goquery.NewDocumentFromReader(strings.NewReader(res)) + _, err := goquery.NewDocumentFromReader(strings.NewReader(buf.String())) require.NoError(t, err) - assert.Equal(t, "

Tags: Bar, Foo

", res) + assert.Equal(t, "

Tags: Bar, Foo

", buf.String()) } func Test_renderOldContentWarning(t *testing.T) { diff --git a/webmentionVerification.go b/webmentionVerification.go index c4fbddd..e37f31c 100644 --- a/webmentionVerification.go +++ b/webmentionVerification.go @@ -11,42 +11,51 @@ import ( "net/http" "net/url" "strings" + "sync" "time" "github.com/PuerkitoBio/goquery" "github.com/thoas/go-funk" + "go.goblog.app/app/pkgs/bufferpool" "go.goblog.app/app/pkgs/contenttype" "willnorris.com/go/microformats" ) func (a *goBlog) initWebmentionQueue() { go func() { - for { + done := false + var wg sync.WaitGroup + wg.Add(1) + a.shutdown.Add(func() { + done = true + wg.Wait() + log.Println("Stopped webmention queue") + }) + for !done { qi, err := a.db.peekQueue("wm") if err != nil { log.Println("webmention queue:", err.Error()) continue - } else if qi != nil { - var m mention - err = gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&m) - if err != nil { - log.Println("webmention queue:", err.Error()) - _ = a.db.dequeue(qi) - continue - } - err = a.verifyMention(&m) - if err != nil { - log.Println(fmt.Sprintf("Failed to verify webmention from %s to %s: %s", m.Source, m.Target, err.Error())) - } - err = a.db.dequeue(qi) - if err != nil { - log.Println("webmention queue:", err.Error()) - } - } else { + } + if qi == nil { // No item in the queue, wait a moment - time.Sleep(15 * time.Second) + time.Sleep(5 * time.Second) + continue + } + var m mention + if err = gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&m); err != nil { + log.Println("webmention queue:", err.Error()) + _ = a.db.dequeue(qi) + continue + } + if err = a.verifyMention(&m); err != nil { + log.Println(fmt.Sprintf("Failed to verify webmention from %s to %s: %s", m.Source, m.Target, err.Error())) + } + if err = a.db.dequeue(qi); err != nil { + log.Println("webmention queue:", err.Error()) } } + wg.Done() }() } @@ -54,8 +63,9 @@ func (a *goBlog) queueMention(m *mention) error { if wm := a.cfg.Webmention; wm != nil && wm.DisableReceiving { return errors.New("webmention receiving disabled") } - var buf bytes.Buffer - if err := gob.NewEncoder(&buf).Encode(m); err != nil { + buf := bufferpool.Get() + defer bufferpool.Put(buf) + if err := gob.NewEncoder(buf).Encode(m); err != nil { return err } return a.db.enqueue("wm", buf.Bytes(), time.Now()) @@ -165,12 +175,13 @@ func (a *goBlog) verifyMention(m *mention) error { } func (a *goBlog) verifyReader(m *mention, body io.Reader) error { - var linksBuffer, gqBuffer, mfBuffer bytes.Buffer - if _, err := io.Copy(io.MultiWriter(&linksBuffer, &gqBuffer, &mfBuffer), body); err != nil { + linksBuffer, gqBuffer, mfBuffer := bufferpool.Get(), bufferpool.Get(), bufferpool.Get() + defer bufferpool.Put(linksBuffer, gqBuffer, mfBuffer) + if _, err := io.Copy(io.MultiWriter(linksBuffer, gqBuffer, mfBuffer), body); err != nil { return err } // Check if source mentions target - links, err := allLinksFromHTML(&linksBuffer, defaultIfEmpty(m.NewSource, m.Source)) + links, err := allLinksFromHTML(linksBuffer, defaultIfEmpty(m.NewSource, m.Source)) if err != nil { return err } @@ -210,13 +221,13 @@ func (a *goBlog) verifyReader(m *mention, body io.Reader) error { m.Author = "" m.Url = "" m.hasUrl = false - m.fillFromData(microformats.Parse(&mfBuffer, sourceURL)) + m.fillFromData(microformats.Parse(mfBuffer, sourceURL)) if m.Url == "" { m.Url = m.Source } // Set title when content is empty as well if m.Title == "" && m.Content == "" { - doc, err := goquery.NewDocumentFromReader(&gqBuffer) + doc, err := goquery.NewDocumentFromReader(gqBuffer) if err != nil { return err }