From 222c792908bf21bda5ffd3842364043875d38a44 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Else Date: Sat, 12 Feb 2022 22:29:45 +0100 Subject: [PATCH] More improvements (less buffers, some fixes) --- activityPub.go | 5 ++++- cache.go | 14 ++++++------ editor.go | 30 ++++++++++++-------------- feeds.go | 23 ++++++++++---------- go.mod | 2 +- go.sum | 3 ++- opensearch.go | 55 +++++++++++++++++++++++++++++++++++++---------- privateMode.go | 5 +---- render.go | 5 +---- robotstxt.go | 9 ++++++-- robotstxt_test.go | 20 ++++++++++------- sitemap.go | 22 ++++++++++++------- 12 files changed, 120 insertions(+), 73 deletions(-) diff --git a/activityPub.go b/activityPub.go index e207671..d0972cb 100644 --- a/activityPub.go +++ b/activityPub.go @@ -7,8 +7,10 @@ import ( "database/sql" "encoding/json" "encoding/pem" + "encoding/xml" "errors" "fmt" + "io" "log" "net/http" "net/url" @@ -242,7 +244,8 @@ func (a *goBlog) apVerifySignature(r *http.Request) (*asPerson, string, int, err func handleWellKnownHostMeta(w http.ResponseWriter, r *http.Request) { w.Header().Set(contentType, "application/xrd+xml"+contenttype.CharsetUtf8Suffix) - _, _ = w.Write([]byte(``)) + _, _ = io.WriteString(w, xml.Header) + _, _ = io.WriteString(w, ``) } func (a *goBlog) apGetRemoteActor(iri string) (*asPerson, int, error) { diff --git a/cache.go b/cache.go index 7f8de71..8448af7 100644 --- a/cache.go +++ b/cache.go @@ -179,21 +179,23 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item * // Record request recorder := httptest.NewRecorder() next.ServeHTTP(recorder, cr) - // Cache values from recorder + recorder.Flush() + // Cache result result := recorder.Result() eTag := sha256.New() body, _ := io.ReadAll(io.TeeReader(result.Body, eTag)) + headers := result.Header.Clone() _ = result.Body.Close() lastMod := time.Now() - if lm := result.Header.Get(lastModified); lm != "" { + if lm := headers.Get(lastModified); lm != "" { if parsedTime, te := dateparse.ParseLocal(lm); te == nil { lastMod = parsedTime } } // Remove problematic headers - result.Header.Del("Accept-Ranges") - result.Header.Del("ETag") - result.Header.Del(lastModified) + headers.Del("Accept-Ranges") + headers.Del("ETag") + headers.Del(lastModified) // Create cache item exp, _ := cr.Context().Value(cacheExpirationKey).(int) item = &cacheItem{ @@ -201,7 +203,7 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item * creationTime: lastMod, eTag: fmt.Sprintf("%x", eTag.Sum(nil)), code: result.StatusCode, - header: result.Header, + header: headers, body: body, } // Save cache diff --git a/editor.go b/editor.go index cb48acd..d7afef5 100644 --- a/editor.go +++ b/editor.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "go.goblog.app/app/pkgs/bufferpool" "go.goblog.app/app/pkgs/contenttype" "gopkg.in/yaml.v3" ws "nhooyr.io/websocket" @@ -93,22 +92,21 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) { }, }) case "updatepost": - jsonBuf := bufferpool.Get() - defer bufferpool.Put(jsonBuf) - err := json.NewEncoder(jsonBuf).Encode(map[string]interface{}{ - "action": actionUpdate, - "url": r.FormValue("url"), - "replace": map[string][]string{ - "content": { - r.FormValue("content"), + pipeReader, pipeWriter := io.Pipe() + defer pipeReader.Close() + go func() { + writeErr := json.NewEncoder(pipeWriter).Encode(map[string]interface{}{ + "action": actionUpdate, + "url": r.FormValue("url"), + "replace": map[string][]string{ + "content": { + r.FormValue("content"), + }, }, - }, - }) - if err != nil { - a.serveError(w, r, err.Error(), http.StatusInternalServerError) - return - } - req, err := http.NewRequest(http.MethodPost, "", jsonBuf) + }) + _ = pipeWriter.CloseWithError(writeErr) + }() + req, err := http.NewRequest(http.MethodPost, "", pipeReader) if err != nil { a.serveError(w, r, err.Error(), http.StatusInternalServerError) return diff --git a/feeds.go b/feeds.go index c9e7466..5224975 100644 --- a/feeds.go +++ b/feeds.go @@ -1,7 +1,7 @@ package main import ( - "bytes" + "io" "net/http" "strings" "time" @@ -52,27 +52,28 @@ func (a *goBlog) generateFeed(blog string, f feedType, w http.ResponseWriter, r }) bufferpool.Put(buf) } - var err error - var feedBuffer bytes.Buffer + var feedWriteFunc func(w io.Writer) error var feedMediaType string switch f { case rssFeed: feedMediaType = contenttype.RSS - err = feed.WriteRss(&feedBuffer) + feedWriteFunc = feed.WriteRss case atomFeed: feedMediaType = contenttype.ATOM - err = feed.WriteAtom(&feedBuffer) + feedWriteFunc = feed.WriteAtom case jsonFeed: feedMediaType = contenttype.JSONFeed - err = feed.WriteJSON(&feedBuffer) + feedWriteFunc = feed.WriteJSON default: a.serve404(w, r) return } - if err != nil { - a.serveError(w, r, err.Error(), http.StatusInternalServerError) - return - } + pipeReader, pipeWriter := io.Pipe() + go func() { + writeErr := feedWriteFunc(pipeWriter) + _ = pipeWriter.CloseWithError(writeErr) + }() w.Header().Set(contentType, feedMediaType+contenttype.CharsetUtf8Suffix) - _ = a.min.Minify(feedMediaType, w, &feedBuffer) + minifyErr := a.min.Minify(feedMediaType, w, pipeReader) + _ = pipeReader.CloseWithError(minifyErr) } diff --git a/go.mod b/go.mod index 7ee3821..7174c80 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/tkrajina/gpxgo v1.2.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 github.com/vcraescu/go-paginator v1.0.1-0.20201114172518-2cfc59fe05c2 - github.com/yuin/goldmark v1.4.5 + github.com/yuin/goldmark v1.4.6 // master github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38 github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594 diff --git a/go.sum b/go.sum index 291e85f..96d291b 100644 --- a/go.sum +++ b/go.sum @@ -447,8 +447,9 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.7/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.5 h1:4OEQwtW2uLXjEdgnGM3Vg652Pq37X7NOIRzFWb3BzIc= github.com/yuin/goldmark v1.4.5/go.mod h1:rmuwmfZ0+bvzB24eSC//bk1R1Zp3hM0OXYv/G2LIilg= +github.com/yuin/goldmark v1.4.6 h1:EQ1OkiNq/eMbQxs/2O/A8VDIHERXGH14s19ednd4XIw= +github.com/yuin/goldmark v1.4.6/go.mod h1:rmuwmfZ0+bvzB24eSC//bk1R1Zp3hM0OXYv/G2LIilg= github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38 h1:XZjLcLoTPNZuxppY3gwhRqo/T2XF6JMGFFdkAjX3w1w= github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38/go.mod h1:RhP/RWpexdp+KHs7ghKnifRoIs/Bq4nDS7tRbCkOwKY= github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594 h1:yHfZyN55+5dp1wG7wDKv8HQ044moxkyGq12KFFMFDxg= diff --git a/opensearch.go b/opensearch.go index 8a31723..3008794 100644 --- a/opensearch.go +++ b/opensearch.go @@ -1,26 +1,59 @@ package main import ( - "bytes" - "fmt" + "encoding/xml" + "io" "net/http" "go.goblog.app/app/pkgs/contenttype" ) +type openSearchDescription struct { + XMLName xml.Name `xml:"http://a9.com/-/spec/opensearch/1.1/ OpenSearchDescription"` + Text string `xml:",chardata"` + ShortName string `xml:"ShortName"` + Description string `xml:"Description"` + URL *openSearchDescriptionUrl `xml:"Url"` + SearchForm string `xml:"http://www.mozilla.org/2006/browser/search/ SearchForm"` +} + +type openSearchDescriptionUrl struct { + Text string `xml:",chardata"` + Type string `xml:"type,attr"` + Method string `xml:"method,attr"` + Template string `xml:"template,attr"` + Param *openSearchDescriptionUrlParam `xml:"Param"` +} + +type openSearchDescriptionUrlParam struct { + Text string `xml:",chardata"` + Name string `xml:"name,attr"` + Value string `xml:"value,attr"` +} + func (a *goBlog) serveOpenSearch(w http.ResponseWriter, r *http.Request) { _, b := a.getBlog(r) title := a.renderMdTitle(b.Title) sURL := a.getFullAddress(b.getRelativePath(defaultIfEmpty(b.Search.Path, defaultSearchPath))) - var buf bytes.Buffer - _, _ = fmt.Fprintf(&buf, ""+ - "%s%s"+ - ""+ - "%s"+ - "", - title, title, sURL, sURL) - w.Header().Set(contentType, "application/opensearchdescription+xml") - _ = a.min.Minify(contenttype.XML, w, &buf) + w.Header().Set(contentType, "application/opensearchdescription+xml"+contenttype.CharsetUtf8Suffix) + openSearch := &openSearchDescription{ + ShortName: title, + Description: title, + URL: &openSearchDescriptionUrl{ + Type: "text/html", + Method: "post", + Template: sURL, + Param: &openSearchDescriptionUrlParam{ + Name: "q", + Value: "{searchTerms}", + }, + }, + SearchForm: sURL, + } + mw := a.min.Writer(contenttype.XML, w) + _, _ = io.WriteString(mw, xml.Header) + _ = xml.NewEncoder(mw).Encode(openSearch) + _ = mw.Close() } func openSearchUrl(b *configBlog) string { diff --git a/privateMode.go b/privateMode.go index aeaa9cb..f1beb99 100644 --- a/privateMode.go +++ b/privateMode.go @@ -7,10 +7,7 @@ import ( ) func (a *goBlog) isPrivate() bool { - if pm := a.cfg.PrivateMode; pm != nil && pm.Enabled { - return true - } - return false + return a.cfg.PrivateMode != nil && a.cfg.PrivateMode.Enabled } func (a *goBlog) privateModeHandler(next http.Handler) http.Handler { diff --git a/render.go b/render.go index cc477bf..a6d2d9b 100644 --- a/render.go +++ b/render.go @@ -1,7 +1,6 @@ package main import ( - "bufio" "io" "net/http" @@ -42,11 +41,9 @@ func (a *goBlog) renderWithStatusCode(w http.ResponseWriter, r *http.Request, st // Render pipeReader, pipeWriter := io.Pipe() go func() { - bufferedPipeWriter := bufio.NewWriter(pipeWriter) - minifyWriter := a.min.Writer(contenttype.HTML, bufferedPipeWriter) + minifyWriter := a.min.Writer(contenttype.HTML, pipeWriter) f(newHtmlBuilder(minifyWriter), data) _ = minifyWriter.Close() - _ = bufferedPipeWriter.Flush() _ = pipeWriter.Close() }() _, readErr := io.Copy(w, pipeReader) diff --git a/robotstxt.go b/robotstxt.go index 2a654fd..222798b 100644 --- a/robotstxt.go +++ b/robotstxt.go @@ -8,9 +8,14 @@ import ( const robotsTXTPath = "/robots.txt" func (a *goBlog) serveRobotsTXT(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, "User-agent: *\n") if a.isPrivate() { - _, _ = w.Write([]byte("User-agent: *\nDisallow: /")) + _, _ = fmt.Fprint(w, "Disallow: /\n") return } - _, _ = w.Write([]byte(fmt.Sprintf("User-agent: *\nSitemap: %v", a.getFullAddress(sitemapPath)))) + _, _ = fmt.Fprint(w, "Allow: /\n\n") + _, _ = fmt.Fprintf(w, "Sitemap: %s\n", a.getFullAddress(sitemapPath)) + for _, bc := range a.cfg.Blogs { + _, _ = fmt.Fprintf(w, "Sitemap: %s\n", a.getFullAddress(bc.getRelativePath(sitemapBlogPath))) + } } diff --git a/robotstxt_test.go b/robotstxt_test.go index f79deb8..581fc6e 100644 --- a/robotstxt_test.go +++ b/robotstxt_test.go @@ -2,6 +2,7 @@ package main import ( "net/http" + "net/http/httptest" "testing" "github.com/stretchr/testify/assert" @@ -17,18 +18,21 @@ func Test_robotsTXT(t *testing.T) { }, } - h := http.HandlerFunc(app.serveRobotsTXT) - assert.HTTPStatusCode(t, h, http.MethodGet, "", nil, 200) - txt := assert.HTTPBody(h, http.MethodGet, "", nil) - assert.Equal(t, "User-agent: *\nSitemap: https://example.com/sitemap.xml", txt) + rec := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/robots.txt", nil) + app.serveRobotsTXT(rec, req) + assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "User-agent: *\nAllow: /\n\nSitemap: https://example.com/sitemap.xml\n", rec.Body.String()) app.cfg.PrivateMode = &configPrivateMode{ Enabled: true, } + assert.True(t, app.isPrivate()) - h = http.HandlerFunc(app.serveRobotsTXT) - assert.HTTPStatusCode(t, h, http.MethodGet, "", nil, 200) - txt = assert.HTTPBody(h, http.MethodGet, "", nil) - assert.Equal(t, "User-agent: *\nDisallow: /", txt) + rec = httptest.NewRecorder() + req = httptest.NewRequest("GET", "/robots.txt", nil) + app.serveRobotsTXT(rec, req) + assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "User-agent: *\nDisallow: /\n", rec.Body.String()) } diff --git a/sitemap.go b/sitemap.go index 36f10fe..99280b0 100644 --- a/sitemap.go +++ b/sitemap.go @@ -1,9 +1,9 @@ package main import ( - "bytes" "database/sql" "encoding/xml" + "io" "net/http" "time" @@ -174,13 +174,19 @@ func (a *goBlog) serveSitemapBlogPosts(w http.ResponseWriter, r *http.Request) { func (a *goBlog) writeSitemapXML(w http.ResponseWriter, sm interface{}) { w.Header().Set(contentType, contenttype.XMLUTF8) - var buf bytes.Buffer - buf.WriteString(xml.Header) - buf.WriteString(``) - _ = xml.NewEncoder(&buf).Encode(sm) - _ = a.min.Minify(contenttype.XML, w, &buf) + pipeReader, pipeWriter := io.Pipe() + go func() { + mw := a.min.Writer(contenttype.XML, pipeWriter) + _, _ = io.WriteString(mw, xml.Header) + _, _ = io.WriteString(mw, ``) + writeErr := xml.NewEncoder(mw).Encode(sm) + _ = mw.Close() + _ = pipeWriter.CloseWithError(writeErr) + }() + _, copyErr := io.Copy(w, pipeReader) + _ = pipeReader.CloseWithError(copyErr) } const sitemapDatePathsSql = `