From ffe70656219a6faf07d3bb7cc19dbd38be00da6b Mon Sep 17 00:00:00 2001 From: Jan-Lukas Else Date: Tue, 24 Jan 2023 09:40:54 +0100 Subject: [PATCH] Replace some buffers with pipes (should reduce memory usage) --- blogroll.go | 22 ++++++++++------------ feeds.go | 6 ++---- geoMap.go | 27 ++++++++++----------------- indieAuthServer.go | 38 ++++++++++++++++---------------------- markdown.go | 27 ++++++++++++--------------- microformats.go | 16 ++++++++-------- 6 files changed, 58 insertions(+), 78 deletions(-) diff --git a/blogroll.go b/blogroll.go index 098a2be..aa3e35e 100644 --- a/blogroll.go +++ b/blogroll.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "io" "log" "net/http" "sort" @@ -51,19 +52,16 @@ func (a *goBlog) serveBlogrollExport(w http.ResponseWriter, r *http.Request) { a.serveError(w, r, "", http.StatusInternalServerError) return } - opmlBuf := bufferpool.Get() - defer bufferpool.Put(opmlBuf) - if err = opml.Render(opmlBuf, &opml.OPML{ - Version: "2.0", - DateCreated: time.Now().UTC(), - Outlines: outlines.([]*opml.Outline), - }); err != nil { - log.Printf("Failed to render OPML: %v", err) - a.serveError(w, r, "", http.StatusInternalServerError) - return - } + pr, pw := io.Pipe() + go func() { + _ = pw.CloseWithError(opml.Render(pw, &opml.OPML{ + Version: "2.0", + DateCreated: time.Now().UTC(), + Outlines: outlines.([]*opml.Outline), + })) + }() w.Header().Set(contentType, contenttype.XMLUTF8) - _ = a.min.Get().Minify(contenttype.XML, w, opmlBuf) + _ = pr.CloseWithError(a.min.Get().Minify(contenttype.XML, w, pr)) } func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) { diff --git a/feeds.go b/feeds.go index e1114d3..2632545 100644 --- a/feeds.go +++ b/feeds.go @@ -78,10 +78,8 @@ func (a *goBlog) generateFeed(blog string, f feedType, w http.ResponseWriter, r } pipeReader, pipeWriter := io.Pipe() go func() { - writeErr := feedWriteFunc(pipeWriter) - _ = pipeWriter.CloseWithError(writeErr) + _ = pipeWriter.CloseWithError(feedWriteFunc(pipeWriter)) }() w.Header().Set(contentType, feedMediaType+contenttype.CharsetUtf8Suffix) - minifyErr := a.min.Get().Minify(feedMediaType, w, pipeReader) - _ = pipeReader.CloseWithError(minifyErr) + _ = pipeReader.CloseWithError(a.min.Get().Minify(feedMediaType, w, pipeReader)) } diff --git a/geoMap.go b/geoMap.go index 14e3389..65e4189 100644 --- a/geoMap.go +++ b/geoMap.go @@ -5,7 +5,6 @@ import ( "io" "net/http" - "go.goblog.app/app/pkgs/bufferpool" "go.goblog.app/app/pkgs/contenttype" ) @@ -89,15 +88,12 @@ func (a *goBlog) serveGeoMapTracks(w http.ResponseWriter, r *http.Request) { } } - buf := bufferpool.Get() - defer bufferpool.Put(buf) - err = json.NewEncoder(buf).Encode(tracks) - if err != nil { - a.serveError(w, r, "", http.StatusInternalServerError) - return - } + pr, pw := io.Pipe() + go func() { + _ = pw.CloseWithError(json.NewEncoder(pw).Encode(tracks)) + }() w.Header().Set(contentType, contenttype.JSONUTF8) - _, _ = io.Copy(w, buf) + _ = pr.CloseWithError(a.min.Get().Minify(contenttype.JSON, w, pr)) } const geoMapLocationsSubpath = "/locations.json" @@ -135,13 +131,10 @@ func (a *goBlog) serveGeoMapLocations(w http.ResponseWriter, r *http.Request) { } } - buf := bufferpool.Get() - defer bufferpool.Put(buf) - err = json.NewEncoder(buf).Encode(locations) - if err != nil { - a.serveError(w, r, "", http.StatusInternalServerError) - return - } + pr, pw := io.Pipe() + go func() { + _ = pw.CloseWithError(json.NewEncoder(pw).Encode(locations)) + }() w.Header().Set(contentType, contenttype.JSONUTF8) - _, _ = io.Copy(w, buf) + _ = pr.CloseWithError(a.min.Get().Minify(contenttype.JSON, w, pr)) } diff --git a/indieAuthServer.go b/indieAuthServer.go index 80cf36c..e9eb694 100644 --- a/indieAuthServer.go +++ b/indieAuthServer.go @@ -4,6 +4,7 @@ import ( "database/sql" "encoding/json" "errors" + "io" "net/http" "net/url" "strings" @@ -11,7 +12,6 @@ import ( "github.com/google/uuid" "github.com/hacdias/indieauth/v2" - "go.goblog.app/app/pkgs/bufferpool" "go.goblog.app/app/pkgs/contenttype" ) @@ -44,14 +44,12 @@ func (a *goBlog) indieAuthMetadata(w http.ResponseWriter, r *http.Request) { "scopes_supported": []string{"create", "update", "delete", "undelete", "media"}, "code_challenge_methods_supported": indieauth.CodeChallengeMethods, } - buf := bufferpool.Get() - defer bufferpool.Put(buf) - if err := json.NewEncoder(buf).Encode(resp); err != nil { - a.serveError(w, r, "Encoding failed", http.StatusInternalServerError) - return - } + pr, pw := io.Pipe() + go func() { + _ = pw.CloseWithError(json.NewEncoder(pw).Encode(resp)) + }() w.Header().Set(contentType, contenttype.JSONUTF8) - _ = a.min.Get().Minify(contenttype.JSON, w, buf) + _ = pr.CloseWithError(a.min.Get().Minify(contenttype.JSON, w, pr)) } // Parse Authorization Request @@ -168,14 +166,12 @@ func (a *goBlog) indieAuthVerification(w http.ResponseWriter, r *http.Request, w resp["access_token"] = token resp["scope"] = strings.Join(data.Scopes, " ") } - buf := bufferpool.Get() - defer bufferpool.Put(buf) - if err = json.NewEncoder(buf).Encode(resp); err != nil { - a.serveError(w, r, "Encoding failed", http.StatusInternalServerError) - return - } + pr, pw := io.Pipe() + go func() { + _ = pw.CloseWithError(json.NewEncoder(pw).Encode(resp)) + }() w.Header().Set(contentType, contenttype.JSONUTF8) - _ = a.min.Get().Minify(contenttype.JSON, w, buf) + _ = pr.CloseWithError(a.min.Get().Minify(contenttype.JSON, w, pr)) } // Save the authorization request and return the code @@ -236,14 +232,12 @@ func (a *goBlog) indieAuthTokenVerification(w http.ResponseWriter, r *http.Reque "scope": strings.Join(data.Scopes, " "), } } - buf := bufferpool.Get() - defer bufferpool.Put(buf) - if err = json.NewEncoder(buf).Encode(res); err != nil { - a.serveError(w, r, "Encoding failed", http.StatusInternalServerError) - return - } + pr, pw := io.Pipe() + go func() { + _ = pw.CloseWithError(json.NewEncoder(pw).Encode(res)) + }() w.Header().Set(contentType, contenttype.JSONUTF8) - _ = a.min.Get().Minify(contenttype.JSON, w, buf) + _ = pr.CloseWithError(a.min.Get().Minify(contenttype.JSON, w, pr)) } // Checks the database for the token and returns the indieAuthData with client and scope. diff --git a/markdown.go b/markdown.go index bbaf1fd..3cbcf0a 100644 --- a/markdown.go +++ b/markdown.go @@ -13,7 +13,6 @@ import ( "github.com/yuin/goldmark/renderer" "github.com/yuin/goldmark/renderer/html" "github.com/yuin/goldmark/util" - "go.goblog.app/app/pkgs/bufferpool" "go.goblog.app/app/pkgs/highlighting" "go.goblog.app/app/pkgs/htmlbuilder" ) @@ -83,13 +82,12 @@ func (a *goBlog) renderText(s string) string { if s == "" { return "" } - buf := bufferpool.Get() - defer bufferpool.Put(buf) - err := a.renderMarkdownToWriter(buf, s, false) - if err != nil { - return "" - } - text, err := htmlTextFromReader(buf) + pr, pw := io.Pipe() + go func() { + _ = pw.CloseWithError(a.renderMarkdownToWriter(pw, s, false)) + }() + text, err := htmlTextFromReader(pr) + _ = pr.CloseWithError(err) if err != nil { return "" } @@ -100,13 +98,12 @@ func (a *goBlog) renderMdTitle(s string) string { if s == "" { return "" } - buf := bufferpool.Get() - defer bufferpool.Put(buf) - err := a.titleMd.Convert([]byte(s), buf) - if err != nil { - return "" - } - text, err := htmlTextFromReader(buf) + pr, pw := io.Pipe() + go func() { + _ = pw.CloseWithError(a.titleMd.Convert([]byte(s), pw)) + }() + text, err := htmlTextFromReader(pr) + _ = pr.CloseWithError(err) if err != nil { return "" } diff --git a/microformats.go b/microformats.go index 4ee4c59..bd23944 100644 --- a/microformats.go +++ b/microformats.go @@ -35,22 +35,22 @@ type microformatsResult struct { } func (a *goBlog) parseMicroformats(u string, cache bool) (*microformatsResult, error) { - buf := bufferpool.Get() - defer bufferpool.Put(buf) + pr, pw := io.Pipe() rb := requests.URL(u). Method(http.MethodGet). Accept(contenttype.HTMLUTF8). Client(a.httpClient). - ToBytesBuffer(buf) + ToWriter(pw) if cache { a.initMicroformatsCache() rb.Transport(httpcachetransport.NewHttpCacheTransport(a.httpClient.Transport, a.mfCache, 10*time.Minute)) } - err := rb.Fetch(context.Background()) - if err != nil { - return nil, err - } - return parseMicroformatsFromReader(u, buf) + go func() { + _ = pw.CloseWithError(rb.Fetch(context.Background())) + }() + result, err := parseMicroformatsFromReader(u, pr) + _ = pr.CloseWithError(err) + return result, err } func parseMicroformatsFromReader(u string, r io.Reader) (*microformatsResult, error) {