Some more small memory improvements

comments.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 
 	"github.com/go-chi/chi/v5"
-	"go.goblog.app/app/pkgs/bufferpool"
+	"go.goblog.app/app/pkgs/builderpool"
 )
 
 const commentPath = "/comment"
@@ -138,8 +138,8 @@ type commentsRequestConfig struct {
 }
 
 func buildCommentsQuery(config *commentsRequestConfig) (query string, args []any) {
-	queryBuilder := bufferpool.Get()
+	queryBuilder := builderpool.Get()
-	defer bufferpool.Put(queryBuilder)
+	defer builderpool.Put(queryBuilder)
 	queryBuilder.WriteString("select id, target, name, website, comment, original from comments")
 	if config.id != 0 {
 		queryBuilder.WriteString(" where id = @id")

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/dmulholl/mp3lib v1.0.0
 	github.com/elnormous/contenttype v1.0.3
 	github.com/emersion/go-smtp v0.16.0
-	github.com/go-ap/activitypub v0.0.0-20221209114049-1ceafda50f9f
+	github.com/go-ap/activitypub v0.0.0-20230203141357-c618b9dc33df
 	github.com/go-ap/client v0.0.0-20230125140507-b30e32f7fa90
 	github.com/go-ap/jsonld v0.0.0-20221030091449-f2a191312c73
 	github.com/go-chi/chi/v5 v5.0.8
@@ -58,7 +58,7 @@ require (
 	github.com/tkrajina/gpxgo v1.2.2-0.20220217201249-321f19554eec
 	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
 	// master
-	github.com/traefik/yaegi v0.14.4-0.20230117132604-1679870ea3c8
+	github.com/traefik/yaegi v0.14.4-0.20230203133205-0e3ea5732a87
 	github.com/vcraescu/go-paginator v1.0.1-0.20201114172518-2cfc59fe05c2
 	github.com/xhit/go-simple-mail/v2 v2.13.0
 	github.com/yuin/goldmark v1.5.4

go.sum

@@ -168,8 +168,8 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm
 github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs=
-github.com/go-ap/activitypub v0.0.0-20221209114049-1ceafda50f9f h1:UV5kupaU8AP8g8Bbsn53q87XCufW/E8wvnTHDKqjoR4=
+github.com/go-ap/activitypub v0.0.0-20230203141357-c618b9dc33df h1:VLPWapyOUOTSK1KzNgAwoZDM7R0/XNdBt53xY8jS898=
-github.com/go-ap/activitypub v0.0.0-20221209114049-1ceafda50f9f/go.mod h1:1oVD0h0aPT3OEE1ZoSUoym/UGKzxe+e0y8K2AkQ1Hqs=
+github.com/go-ap/activitypub v0.0.0-20230203141357-c618b9dc33df/go.mod h1:1oVD0h0aPT3OEE1ZoSUoym/UGKzxe+e0y8K2AkQ1Hqs=
 github.com/go-ap/client v0.0.0-20230125140507-b30e32f7fa90 h1:6RecS9v9M00TAnGpaAj4cRjGVZzFVuqXWoZrefGBFro=
 github.com/go-ap/client v0.0.0-20230125140507-b30e32f7fa90/go.mod h1:P5qNxXHk44o9OEmHxmjCFjS5jLZa4BZv6h7lHrmC1MA=
 github.com/go-ap/errors v0.0.0-20221205040414-01c1adfc98ea h1:ywGtLGVjJjMrq4mu35Qmu+NtlhlTk/gTayE6Bb4tQZk=
@@ -554,8 +554,8 @@ github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJ
 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
 github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 h1:PM5hJF7HVfNWmCjMdEfbuOBNXSVF2cMFGgQTPdKCbwM=
 github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208/go.mod h1:BzWtXXrXzZUvMacR0oF/fbDDgUPO8L36tDMmRAf14ns=
-github.com/traefik/yaegi v0.14.4-0.20230117132604-1679870ea3c8 h1:/7StGZkjdW/GtwISKUGl2hz6TM+0eYYjTCxppbSAgnk=
+github.com/traefik/yaegi v0.14.4-0.20230203133205-0e3ea5732a87 h1:DoLt/8arxW4WLXQ+cdC+N2ezsAJDfycJMyRNsDQJgaU=
-github.com/traefik/yaegi v0.14.4-0.20230117132604-1679870ea3c8/go.mod h1:AVRxhaI2G+nUsaM1zyktzwXn69G3t/AuTDrCiTds9p0=
+github.com/traefik/yaegi v0.14.4-0.20230203133205-0e3ea5732a87/go.mod h1:AVRxhaI2G+nUsaM1zyktzwXn69G3t/AuTDrCiTds9p0=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=

micropubMedia.go

@@ -9,13 +9,13 @@ import (
 	"path/filepath"
 	"strings"
 
-	"go.goblog.app/app/pkgs/bufferpool"
 	"go.goblog.app/app/pkgs/contenttype"
 )
 
 const micropubMediaSubPath = "/media"
 
 func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
 	// Check scope
 	if !a.micropubCheckScope(w, r, "media") {
 		return
@@ -28,22 +28,23 @@ func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) {
 	// Parse multipart form
 	err := r.ParseMultipartForm(0)
 	if err != nil {
-		a.serveError(w, r, err.Error(), http.StatusBadRequest)
+		a.serveError(w, r, "failed to parse multipart form", http.StatusBadRequest)
 		return
 	}
 	// Get file
 	file, header, err := r.FormFile("file")
 	if err != nil {
-		a.serveError(w, r, err.Error(), http.StatusBadRequest)
+		a.serveError(w, r, "failed to get multipart file", http.StatusBadRequest)
 		return
 	}
-	// Read the file into temporary buffer and generate sha256 hash
+	defer file.Close()
+	// Generate sha256 hash for file
 	hash := sha256.New()
-	buffer := bufferpool.Get()
+	_, err = io.Copy(hash, file)
-	defer bufferpool.Put(buffer)
+	if err != nil {
-	_, _ = io.Copy(buffer, io.TeeReader(file, hash))
+		a.serveError(w, r, "failed to get file hash", http.StatusBadRequest)
-	_ = file.Close()
+		return
-	_ = r.Body.Close()
+	}
 	// Get file extension
 	fileExtension := filepath.Ext(header.Filename)
 	if fileExtension == "" {
@@ -59,9 +60,14 @@ func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) {
 	// Generate the file name
 	fileName := fmt.Sprintf("%x%s", hash.Sum(nil), fileExtension)
 	// Save file
-	location, err := a.saveMediaFile(fileName, buffer)
+	_, err = file.Seek(0, io.SeekStart)
 	if err != nil {
-		a.serveError(w, r, "failed to save original file: "+err.Error(), http.StatusInternalServerError)
+		a.serveError(w, r, "failed to read multipart file", http.StatusInternalServerError)
+		return
+	}
+	location, err := a.saveMediaFile(fileName, file)
+	if err != nil {
+		a.serveError(w, r, "failed to save original file", http.StatusInternalServerError)
 		return
 	}
 	// Try to compress file (only when not in private mode)

templateAssets.go

@@ -12,7 +12,6 @@ import (
 	"strings"
 
 	chromahtml "github.com/alecthomas/chroma/v2/formatters/html"
-	"go.goblog.app/app/pkgs/bufferpool"
 	"go.goblog.app/app/pkgs/contenttype"
 	"go.goblog.app/app/pkgs/highlighting"
 )
@@ -114,11 +113,11 @@ func (a *goBlog) initChromaCSS() error {
 		return err
 	}
 	// Generate and minify CSS
-	buf := bufferpool.Get()
+	pr, pw := io.Pipe()
-	defer bufferpool.Put(buf)
+	go func() {
-	err = chromahtml.New(chromahtml.ClassPrefix("c-")).WriteCSS(buf, chromaStyle)
+		_ = pw.CloseWithError(chromahtml.New(chromahtml.ClassPrefix("c-")).WriteCSS(pw, chromaStyle))
-	if err != nil {
+	}()
-		return err
+	err = a.compileAsset(chromaPath, pr)
-	}
+	_ = pr.CloseWithError(err)
-	return a.compileAsset(chromaPath, buf)
+	return err
 }

webmentionSending.go

@@ -14,7 +14,6 @@ import (
 	"github.com/carlmjohnson/requests"
 	"github.com/samber/lo"
 	"github.com/tomnomnom/linkheader"
-	"go.goblog.app/app/pkgs/bufferpool"
 )
 
 const postParamWebmention = "webmention"
@@ -32,10 +31,13 @@ func (a *goBlog) sendWebmentions(p *post) error {
 		// Ignore this post
 		return nil
 	}
-	contentBuf := bufferpool.Get()
+	pr, pw := io.Pipe()
-	a.postHtmlToWriter(contentBuf, &postHtmlOptions{p: p})
+	go func() {
-	links, err := allLinksFromHTML(contentBuf, a.fullPostURL(p))
+		a.postHtmlToWriter(pw, &postHtmlOptions{p: p})
-	bufferpool.Put(contentBuf)
+		_ = pw.Close()
+	}()
+	links, err := allLinksFromHTML(pr, a.fullPostURL(p))
+	_ = pr.CloseWithError(err)
 	if err != nil {
 		return err
 	}

webmentionVerification.go

@@ -141,13 +141,16 @@ func (a *goBlog) verifyMention(m *mention) error {
 }
 
 func (a *goBlog) verifyReader(m *mention, body io.Reader) error {
-	linksBuffer, mfBuffer := bufferpool.Get(), bufferpool.Get()
+	mfBuffer := bufferpool.Get()
-	defer bufferpool.Put(linksBuffer, mfBuffer)
+	defer bufferpool.Put(mfBuffer)
-	if _, err := io.Copy(io.MultiWriter(linksBuffer, mfBuffer), body); err != nil {
+	pr, pw := io.Pipe()
-		return err
+	go func() {
-	}
+		_, err := io.Copy(io.MultiWriter(pw, mfBuffer), body)
+		_ = pw.CloseWithError(err)
+	}()
 	// Check if source mentions target
-	links, err := allLinksFromHTML(linksBuffer, defaultIfEmpty(m.NewSource, m.Source))
+	links, err := allLinksFromHTML(pr, defaultIfEmpty(m.NewSource, m.Source))
+	_ = pr.CloseWithError(err)
 	if err != nil {
 		return err
 	}