Replace some buffers with pipes (should reduce memory usage)

blogroll.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"io"
 	"log"
 	"net/http"
 	"sort"
@@ -51,19 +52,16 @@ func (a *goBlog) serveBlogrollExport(w http.ResponseWriter, r *http.Request) {
 		a.serveError(w, r, "", http.StatusInternalServerError)
 		return
 	}
-	opmlBuf := bufferpool.Get()
+	pr, pw := io.Pipe()
-	defer bufferpool.Put(opmlBuf)
+	go func() {
-	if err = opml.Render(opmlBuf, &opml.OPML{
+		_ = pw.CloseWithError(opml.Render(pw, &opml.OPML{
-		Version:     "2.0",
+			Version:     "2.0",
-		DateCreated: time.Now().UTC(),
+			DateCreated: time.Now().UTC(),
-		Outlines:    outlines.([]*opml.Outline),
+			Outlines:    outlines.([]*opml.Outline),
-	}); err != nil {
+		}))
-		log.Printf("Failed to render OPML: %v", err)
+	}()
-		a.serveError(w, r, "", http.StatusInternalServerError)
-		return
-	}
 	w.Header().Set(contentType, contenttype.XMLUTF8)
-	_ = a.min.Get().Minify(contenttype.XML, w, opmlBuf)
+	_ = pr.CloseWithError(a.min.Get().Minify(contenttype.XML, w, pr))
 }
 
 func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) {

feeds.go

@@ -78,10 +78,8 @@ func (a *goBlog) generateFeed(blog string, f feedType, w http.ResponseWriter, r
 	}
 	pipeReader, pipeWriter := io.Pipe()
 	go func() {
-		writeErr := feedWriteFunc(pipeWriter)
+		_ = pipeWriter.CloseWithError(feedWriteFunc(pipeWriter))
-		_ = pipeWriter.CloseWithError(writeErr)
 	}()
 	w.Header().Set(contentType, feedMediaType+contenttype.CharsetUtf8Suffix)
-	minifyErr := a.min.Get().Minify(feedMediaType, w, pipeReader)
+	_ = pipeReader.CloseWithError(a.min.Get().Minify(feedMediaType, w, pipeReader))
-	_ = pipeReader.CloseWithError(minifyErr)
 }

geoMap.go

@@ -5,7 +5,6 @@ import (
 	"io"
 	"net/http"
 
-	"go.goblog.app/app/pkgs/bufferpool"
 	"go.goblog.app/app/pkgs/contenttype"
 )
 
@@ -89,15 +88,12 @@ func (a *goBlog) serveGeoMapTracks(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	buf := bufferpool.Get()
+	pr, pw := io.Pipe()
-	defer bufferpool.Put(buf)
+	go func() {
-	err = json.NewEncoder(buf).Encode(tracks)
+		_ = pw.CloseWithError(json.NewEncoder(pw).Encode(tracks))
-	if err != nil {
+	}()
-		a.serveError(w, r, "", http.StatusInternalServerError)
-		return
-	}
 	w.Header().Set(contentType, contenttype.JSONUTF8)
-	_, _ = io.Copy(w, buf)
+	_ = pr.CloseWithError(a.min.Get().Minify(contenttype.JSON, w, pr))
 }
 
 const geoMapLocationsSubpath = "/locations.json"
@@ -135,13 +131,10 @@ func (a *goBlog) serveGeoMapLocations(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	buf := bufferpool.Get()
+	pr, pw := io.Pipe()
-	defer bufferpool.Put(buf)
+	go func() {
-	err = json.NewEncoder(buf).Encode(locations)
+		_ = pw.CloseWithError(json.NewEncoder(pw).Encode(locations))
-	if err != nil {
+	}()
-		a.serveError(w, r, "", http.StatusInternalServerError)
-		return
-	}
 	w.Header().Set(contentType, contenttype.JSONUTF8)
-	_, _ = io.Copy(w, buf)
+	_ = pr.CloseWithError(a.min.Get().Minify(contenttype.JSON, w, pr))
 }

indieAuthServer.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"encoding/json"
 	"errors"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -11,7 +12,6 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/hacdias/indieauth/v2"
-	"go.goblog.app/app/pkgs/bufferpool"
 	"go.goblog.app/app/pkgs/contenttype"
 )
 
@@ -44,14 +44,12 @@ func (a *goBlog) indieAuthMetadata(w http.ResponseWriter, r *http.Request) {
 		"scopes_supported":                           []string{"create", "update", "delete", "undelete", "media"},
 		"code_challenge_methods_supported":           indieauth.CodeChallengeMethods,
 	}
-	buf := bufferpool.Get()
+	pr, pw := io.Pipe()
-	defer bufferpool.Put(buf)
+	go func() {
-	if err := json.NewEncoder(buf).Encode(resp); err != nil {
+		_ = pw.CloseWithError(json.NewEncoder(pw).Encode(resp))
-		a.serveError(w, r, "Encoding failed", http.StatusInternalServerError)
+	}()
-		return
-	}
 	w.Header().Set(contentType, contenttype.JSONUTF8)
-	_ = a.min.Get().Minify(contenttype.JSON, w, buf)
+	_ = pr.CloseWithError(a.min.Get().Minify(contenttype.JSON, w, pr))
 }
 
 // Parse Authorization Request
@@ -168,14 +166,12 @@ func (a *goBlog) indieAuthVerification(w http.ResponseWriter, r *http.Request, w
 		resp["access_token"] = token
 		resp["scope"] = strings.Join(data.Scopes, " ")
 	}
-	buf := bufferpool.Get()
+	pr, pw := io.Pipe()
-	defer bufferpool.Put(buf)
+	go func() {
-	if err = json.NewEncoder(buf).Encode(resp); err != nil {
+		_ = pw.CloseWithError(json.NewEncoder(pw).Encode(resp))
-		a.serveError(w, r, "Encoding failed", http.StatusInternalServerError)
+	}()
-		return
-	}
 	w.Header().Set(contentType, contenttype.JSONUTF8)
-	_ = a.min.Get().Minify(contenttype.JSON, w, buf)
+	_ = pr.CloseWithError(a.min.Get().Minify(contenttype.JSON, w, pr))
 }
 
 // Save the authorization request and return the code
@@ -236,14 +232,12 @@ func (a *goBlog) indieAuthTokenVerification(w http.ResponseWriter, r *http.Reque
 			"scope":     strings.Join(data.Scopes, " "),
 		}
 	}
-	buf := bufferpool.Get()
+	pr, pw := io.Pipe()
-	defer bufferpool.Put(buf)
+	go func() {
-	if err = json.NewEncoder(buf).Encode(res); err != nil {
+		_ = pw.CloseWithError(json.NewEncoder(pw).Encode(res))
-		a.serveError(w, r, "Encoding failed", http.StatusInternalServerError)
+	}()
-		return
-	}
 	w.Header().Set(contentType, contenttype.JSONUTF8)
-	_ = a.min.Get().Minify(contenttype.JSON, w, buf)
+	_ = pr.CloseWithError(a.min.Get().Minify(contenttype.JSON, w, pr))
 }
 
 // Checks the database for the token and returns the indieAuthData with client and scope.

markdown.go

@@ -13,7 +13,6 @@ import (
 	"github.com/yuin/goldmark/renderer"
 	"github.com/yuin/goldmark/renderer/html"
 	"github.com/yuin/goldmark/util"
-	"go.goblog.app/app/pkgs/bufferpool"
 	"go.goblog.app/app/pkgs/highlighting"
 	"go.goblog.app/app/pkgs/htmlbuilder"
 )
@@ -83,13 +82,12 @@ func (a *goBlog) renderText(s string) string {
 	if s == "" {
 		return ""
 	}
-	buf := bufferpool.Get()
+	pr, pw := io.Pipe()
-	defer bufferpool.Put(buf)
+	go func() {
-	err := a.renderMarkdownToWriter(buf, s, false)
+		_ = pw.CloseWithError(a.renderMarkdownToWriter(pw, s, false))
-	if err != nil {
+	}()
-		return ""
+	text, err := htmlTextFromReader(pr)
-	}
+	_ = pr.CloseWithError(err)
-	text, err := htmlTextFromReader(buf)
 	if err != nil {
 		return ""
 	}
@@ -100,13 +98,12 @@ func (a *goBlog) renderMdTitle(s string) string {
 	if s == "" {
 		return ""
 	}
-	buf := bufferpool.Get()
+	pr, pw := io.Pipe()
-	defer bufferpool.Put(buf)
+	go func() {
-	err := a.titleMd.Convert([]byte(s), buf)
+		_ = pw.CloseWithError(a.titleMd.Convert([]byte(s), pw))
-	if err != nil {
+	}()
-		return ""
+	text, err := htmlTextFromReader(pr)
-	}
+	_ = pr.CloseWithError(err)
-	text, err := htmlTextFromReader(buf)
 	if err != nil {
 		return ""
 	}

microformats.go

@@ -35,22 +35,22 @@ type microformatsResult struct {
 }
 
 func (a *goBlog) parseMicroformats(u string, cache bool) (*microformatsResult, error) {
-	buf := bufferpool.Get()
+	pr, pw := io.Pipe()
-	defer bufferpool.Put(buf)
 	rb := requests.URL(u).
 		Method(http.MethodGet).
 		Accept(contenttype.HTMLUTF8).
 		Client(a.httpClient).
-		ToBytesBuffer(buf)
+		ToWriter(pw)
 	if cache {
 		a.initMicroformatsCache()
 		rb.Transport(httpcachetransport.NewHttpCacheTransport(a.httpClient.Transport, a.mfCache, 10*time.Minute))
 	}
-	err := rb.Fetch(context.Background())
+	go func() {
-	if err != nil {
+		_ = pw.CloseWithError(rb.Fetch(context.Background()))
-		return nil, err
+	}()
-	}
+	result, err := parseMicroformatsFromReader(u, pr)
-	return parseMicroformatsFromReader(u, buf)
+	_ = pr.CloseWithError(err)
+	return result, err
 }
 
 func parseMicroformatsFromReader(u string, r io.Reader) (*microformatsResult, error) {