More improvements (less buffers, some fixes)

This commit is contained in:
Jan-Lukas Else 2022-02-12 22:29:45 +01:00
parent 51eaf24ff2
commit 222c792908
12 changed files with 120 additions and 73 deletions

View File

@ -7,8 +7,10 @@ import (
"database/sql" "database/sql"
"encoding/json" "encoding/json"
"encoding/pem" "encoding/pem"
"encoding/xml"
"errors" "errors"
"fmt" "fmt"
"io"
"log" "log"
"net/http" "net/http"
"net/url" "net/url"
@ -242,7 +244,8 @@ func (a *goBlog) apVerifySignature(r *http.Request) (*asPerson, string, int, err
func handleWellKnownHostMeta(w http.ResponseWriter, r *http.Request) { func handleWellKnownHostMeta(w http.ResponseWriter, r *http.Request) {
w.Header().Set(contentType, "application/xrd+xml"+contenttype.CharsetUtf8Suffix) w.Header().Set(contentType, "application/xrd+xml"+contenttype.CharsetUtf8Suffix)
_, _ = w.Write([]byte(`<?xml version="1.0" encoding="UTF-8"?><XRD xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0"><Link rel="lrdd" type="application/xrd+xml" template="https://` + r.Host + `/.well-known/webfinger?resource={uri}"/></XRD>`)) _, _ = io.WriteString(w, xml.Header)
_, _ = io.WriteString(w, `<XRD xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0"><Link rel="lrdd" type="application/xrd+xml" template="https://`+r.Host+`/.well-known/webfinger?resource={uri}"/></XRD>`)
} }
func (a *goBlog) apGetRemoteActor(iri string) (*asPerson, int, error) { func (a *goBlog) apGetRemoteActor(iri string) (*asPerson, int, error) {

View File

@ -179,21 +179,23 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *
// Record request // Record request
recorder := httptest.NewRecorder() recorder := httptest.NewRecorder()
next.ServeHTTP(recorder, cr) next.ServeHTTP(recorder, cr)
// Cache values from recorder recorder.Flush()
// Cache result
result := recorder.Result() result := recorder.Result()
eTag := sha256.New() eTag := sha256.New()
body, _ := io.ReadAll(io.TeeReader(result.Body, eTag)) body, _ := io.ReadAll(io.TeeReader(result.Body, eTag))
headers := result.Header.Clone()
_ = result.Body.Close() _ = result.Body.Close()
lastMod := time.Now() lastMod := time.Now()
if lm := result.Header.Get(lastModified); lm != "" { if lm := headers.Get(lastModified); lm != "" {
if parsedTime, te := dateparse.ParseLocal(lm); te == nil { if parsedTime, te := dateparse.ParseLocal(lm); te == nil {
lastMod = parsedTime lastMod = parsedTime
} }
} }
// Remove problematic headers // Remove problematic headers
result.Header.Del("Accept-Ranges") headers.Del("Accept-Ranges")
result.Header.Del("ETag") headers.Del("ETag")
result.Header.Del(lastModified) headers.Del(lastModified)
// Create cache item // Create cache item
exp, _ := cr.Context().Value(cacheExpirationKey).(int) exp, _ := cr.Context().Value(cacheExpirationKey).(int)
item = &cacheItem{ item = &cacheItem{
@ -201,7 +203,7 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *
creationTime: lastMod, creationTime: lastMod,
eTag: fmt.Sprintf("%x", eTag.Sum(nil)), eTag: fmt.Sprintf("%x", eTag.Sum(nil)),
code: result.StatusCode, code: result.StatusCode,
header: result.Header, header: headers,
body: body, body: body,
} }
// Save cache // Save cache

View File

@ -11,7 +11,6 @@ import (
"strings" "strings"
"time" "time"
"go.goblog.app/app/pkgs/bufferpool"
"go.goblog.app/app/pkgs/contenttype" "go.goblog.app/app/pkgs/contenttype"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
ws "nhooyr.io/websocket" ws "nhooyr.io/websocket"
@ -93,22 +92,21 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) {
}, },
}) })
case "updatepost": case "updatepost":
jsonBuf := bufferpool.Get() pipeReader, pipeWriter := io.Pipe()
defer bufferpool.Put(jsonBuf) defer pipeReader.Close()
err := json.NewEncoder(jsonBuf).Encode(map[string]interface{}{ go func() {
"action": actionUpdate, writeErr := json.NewEncoder(pipeWriter).Encode(map[string]interface{}{
"url": r.FormValue("url"), "action": actionUpdate,
"replace": map[string][]string{ "url": r.FormValue("url"),
"content": { "replace": map[string][]string{
r.FormValue("content"), "content": {
r.FormValue("content"),
},
}, },
}, })
}) _ = pipeWriter.CloseWithError(writeErr)
if err != nil { }()
a.serveError(w, r, err.Error(), http.StatusInternalServerError) req, err := http.NewRequest(http.MethodPost, "", pipeReader)
return
}
req, err := http.NewRequest(http.MethodPost, "", jsonBuf)
if err != nil { if err != nil {
a.serveError(w, r, err.Error(), http.StatusInternalServerError) a.serveError(w, r, err.Error(), http.StatusInternalServerError)
return return

View File

@ -1,7 +1,7 @@
package main package main
import ( import (
"bytes" "io"
"net/http" "net/http"
"strings" "strings"
"time" "time"
@ -52,27 +52,28 @@ func (a *goBlog) generateFeed(blog string, f feedType, w http.ResponseWriter, r
}) })
bufferpool.Put(buf) bufferpool.Put(buf)
} }
var err error var feedWriteFunc func(w io.Writer) error
var feedBuffer bytes.Buffer
var feedMediaType string var feedMediaType string
switch f { switch f {
case rssFeed: case rssFeed:
feedMediaType = contenttype.RSS feedMediaType = contenttype.RSS
err = feed.WriteRss(&feedBuffer) feedWriteFunc = feed.WriteRss
case atomFeed: case atomFeed:
feedMediaType = contenttype.ATOM feedMediaType = contenttype.ATOM
err = feed.WriteAtom(&feedBuffer) feedWriteFunc = feed.WriteAtom
case jsonFeed: case jsonFeed:
feedMediaType = contenttype.JSONFeed feedMediaType = contenttype.JSONFeed
err = feed.WriteJSON(&feedBuffer) feedWriteFunc = feed.WriteJSON
default: default:
a.serve404(w, r) a.serve404(w, r)
return return
} }
if err != nil { pipeReader, pipeWriter := io.Pipe()
a.serveError(w, r, err.Error(), http.StatusInternalServerError) go func() {
return writeErr := feedWriteFunc(pipeWriter)
} _ = pipeWriter.CloseWithError(writeErr)
}()
w.Header().Set(contentType, feedMediaType+contenttype.CharsetUtf8Suffix) w.Header().Set(contentType, feedMediaType+contenttype.CharsetUtf8Suffix)
_ = a.min.Minify(feedMediaType, w, &feedBuffer) minifyErr := a.min.Minify(feedMediaType, w, pipeReader)
_ = pipeReader.CloseWithError(minifyErr)
} }

2
go.mod
View File

@ -52,7 +52,7 @@ require (
github.com/tkrajina/gpxgo v1.2.0 github.com/tkrajina/gpxgo v1.2.0
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
github.com/vcraescu/go-paginator v1.0.1-0.20201114172518-2cfc59fe05c2 github.com/vcraescu/go-paginator v1.0.1-0.20201114172518-2cfc59fe05c2
github.com/yuin/goldmark v1.4.5 github.com/yuin/goldmark v1.4.6
// master // master
github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38 github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38
github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594 github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594

3
go.sum
View File

@ -447,8 +447,9 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.7/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.3.7/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.5 h1:4OEQwtW2uLXjEdgnGM3Vg652Pq37X7NOIRzFWb3BzIc=
github.com/yuin/goldmark v1.4.5/go.mod h1:rmuwmfZ0+bvzB24eSC//bk1R1Zp3hM0OXYv/G2LIilg= github.com/yuin/goldmark v1.4.5/go.mod h1:rmuwmfZ0+bvzB24eSC//bk1R1Zp3hM0OXYv/G2LIilg=
github.com/yuin/goldmark v1.4.6 h1:EQ1OkiNq/eMbQxs/2O/A8VDIHERXGH14s19ednd4XIw=
github.com/yuin/goldmark v1.4.6/go.mod h1:rmuwmfZ0+bvzB24eSC//bk1R1Zp3hM0OXYv/G2LIilg=
github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38 h1:XZjLcLoTPNZuxppY3gwhRqo/T2XF6JMGFFdkAjX3w1w= github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38 h1:XZjLcLoTPNZuxppY3gwhRqo/T2XF6JMGFFdkAjX3w1w=
github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38/go.mod h1:RhP/RWpexdp+KHs7ghKnifRoIs/Bq4nDS7tRbCkOwKY= github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38/go.mod h1:RhP/RWpexdp+KHs7ghKnifRoIs/Bq4nDS7tRbCkOwKY=
github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594 h1:yHfZyN55+5dp1wG7wDKv8HQ044moxkyGq12KFFMFDxg= github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594 h1:yHfZyN55+5dp1wG7wDKv8HQ044moxkyGq12KFFMFDxg=

View File

@ -1,26 +1,59 @@
package main package main
import ( import (
"bytes" "encoding/xml"
"fmt" "io"
"net/http" "net/http"
"go.goblog.app/app/pkgs/contenttype" "go.goblog.app/app/pkgs/contenttype"
) )
type openSearchDescription struct {
XMLName xml.Name `xml:"http://a9.com/-/spec/opensearch/1.1/ OpenSearchDescription"`
Text string `xml:",chardata"`
ShortName string `xml:"ShortName"`
Description string `xml:"Description"`
URL *openSearchDescriptionUrl `xml:"Url"`
SearchForm string `xml:"http://www.mozilla.org/2006/browser/search/ SearchForm"`
}
type openSearchDescriptionUrl struct {
Text string `xml:",chardata"`
Type string `xml:"type,attr"`
Method string `xml:"method,attr"`
Template string `xml:"template,attr"`
Param *openSearchDescriptionUrlParam `xml:"Param"`
}
type openSearchDescriptionUrlParam struct {
Text string `xml:",chardata"`
Name string `xml:"name,attr"`
Value string `xml:"value,attr"`
}
func (a *goBlog) serveOpenSearch(w http.ResponseWriter, r *http.Request) { func (a *goBlog) serveOpenSearch(w http.ResponseWriter, r *http.Request) {
_, b := a.getBlog(r) _, b := a.getBlog(r)
title := a.renderMdTitle(b.Title) title := a.renderMdTitle(b.Title)
sURL := a.getFullAddress(b.getRelativePath(defaultIfEmpty(b.Search.Path, defaultSearchPath))) sURL := a.getFullAddress(b.getRelativePath(defaultIfEmpty(b.Search.Path, defaultSearchPath)))
var buf bytes.Buffer w.Header().Set(contentType, "application/opensearchdescription+xml"+contenttype.CharsetUtf8Suffix)
_, _ = fmt.Fprintf(&buf, "<?xml version=\"1.0\"?><OpenSearchDescription xmlns=\"http://a9.com/-/spec/opensearch/1.1/\" xmlns:moz=\"http://www.mozilla.org/2006/browser/search/\">"+ openSearch := &openSearchDescription{
"<ShortName>%s</ShortName><Description>%s</Description>"+ ShortName: title,
"<Url type=\"text/html\" method=\"post\" template=\"%s\"><Param name=\"q\" value=\"{searchTerms}\" /></Url>"+ Description: title,
"<moz:SearchForm>%s</moz:SearchForm>"+ URL: &openSearchDescriptionUrl{
"</OpenSearchDescription>", Type: "text/html",
title, title, sURL, sURL) Method: "post",
w.Header().Set(contentType, "application/opensearchdescription+xml") Template: sURL,
_ = a.min.Minify(contenttype.XML, w, &buf) Param: &openSearchDescriptionUrlParam{
Name: "q",
Value: "{searchTerms}",
},
},
SearchForm: sURL,
}
mw := a.min.Writer(contenttype.XML, w)
_, _ = io.WriteString(mw, xml.Header)
_ = xml.NewEncoder(mw).Encode(openSearch)
_ = mw.Close()
} }
func openSearchUrl(b *configBlog) string { func openSearchUrl(b *configBlog) string {

View File

@ -7,10 +7,7 @@ import (
) )
func (a *goBlog) isPrivate() bool { func (a *goBlog) isPrivate() bool {
if pm := a.cfg.PrivateMode; pm != nil && pm.Enabled { return a.cfg.PrivateMode != nil && a.cfg.PrivateMode.Enabled
return true
}
return false
} }
func (a *goBlog) privateModeHandler(next http.Handler) http.Handler { func (a *goBlog) privateModeHandler(next http.Handler) http.Handler {

View File

@ -1,7 +1,6 @@
package main package main
import ( import (
"bufio"
"io" "io"
"net/http" "net/http"
@ -42,11 +41,9 @@ func (a *goBlog) renderWithStatusCode(w http.ResponseWriter, r *http.Request, st
// Render // Render
pipeReader, pipeWriter := io.Pipe() pipeReader, pipeWriter := io.Pipe()
go func() { go func() {
bufferedPipeWriter := bufio.NewWriter(pipeWriter) minifyWriter := a.min.Writer(contenttype.HTML, pipeWriter)
minifyWriter := a.min.Writer(contenttype.HTML, bufferedPipeWriter)
f(newHtmlBuilder(minifyWriter), data) f(newHtmlBuilder(minifyWriter), data)
_ = minifyWriter.Close() _ = minifyWriter.Close()
_ = bufferedPipeWriter.Flush()
_ = pipeWriter.Close() _ = pipeWriter.Close()
}() }()
_, readErr := io.Copy(w, pipeReader) _, readErr := io.Copy(w, pipeReader)

View File

@ -8,9 +8,14 @@ import (
const robotsTXTPath = "/robots.txt" const robotsTXTPath = "/robots.txt"
func (a *goBlog) serveRobotsTXT(w http.ResponseWriter, r *http.Request) { func (a *goBlog) serveRobotsTXT(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprint(w, "User-agent: *\n")
if a.isPrivate() { if a.isPrivate() {
_, _ = w.Write([]byte("User-agent: *\nDisallow: /")) _, _ = fmt.Fprint(w, "Disallow: /\n")
return return
} }
_, _ = w.Write([]byte(fmt.Sprintf("User-agent: *\nSitemap: %v", a.getFullAddress(sitemapPath)))) _, _ = fmt.Fprint(w, "Allow: /\n\n")
_, _ = fmt.Fprintf(w, "Sitemap: %s\n", a.getFullAddress(sitemapPath))
for _, bc := range a.cfg.Blogs {
_, _ = fmt.Fprintf(w, "Sitemap: %s\n", a.getFullAddress(bc.getRelativePath(sitemapBlogPath)))
}
} }

View File

@ -2,6 +2,7 @@ package main
import ( import (
"net/http" "net/http"
"net/http/httptest"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -17,18 +18,21 @@ func Test_robotsTXT(t *testing.T) {
}, },
} }
h := http.HandlerFunc(app.serveRobotsTXT) rec := httptest.NewRecorder()
assert.HTTPStatusCode(t, h, http.MethodGet, "", nil, 200) req := httptest.NewRequest("GET", "/robots.txt", nil)
txt := assert.HTTPBody(h, http.MethodGet, "", nil) app.serveRobotsTXT(rec, req)
assert.Equal(t, "User-agent: *\nSitemap: https://example.com/sitemap.xml", txt) assert.Equal(t, http.StatusOK, rec.Code)
assert.Equal(t, "User-agent: *\nAllow: /\n\nSitemap: https://example.com/sitemap.xml\n", rec.Body.String())
app.cfg.PrivateMode = &configPrivateMode{ app.cfg.PrivateMode = &configPrivateMode{
Enabled: true, Enabled: true,
} }
assert.True(t, app.isPrivate())
h = http.HandlerFunc(app.serveRobotsTXT) rec = httptest.NewRecorder()
assert.HTTPStatusCode(t, h, http.MethodGet, "", nil, 200) req = httptest.NewRequest("GET", "/robots.txt", nil)
txt = assert.HTTPBody(h, http.MethodGet, "", nil) app.serveRobotsTXT(rec, req)
assert.Equal(t, "User-agent: *\nDisallow: /", txt) assert.Equal(t, http.StatusOK, rec.Code)
assert.Equal(t, "User-agent: *\nDisallow: /\n", rec.Body.String())
} }

View File

@ -1,9 +1,9 @@
package main package main
import ( import (
"bytes"
"database/sql" "database/sql"
"encoding/xml" "encoding/xml"
"io"
"net/http" "net/http"
"time" "time"
@ -174,13 +174,19 @@ func (a *goBlog) serveSitemapBlogPosts(w http.ResponseWriter, r *http.Request) {
func (a *goBlog) writeSitemapXML(w http.ResponseWriter, sm interface{}) { func (a *goBlog) writeSitemapXML(w http.ResponseWriter, sm interface{}) {
w.Header().Set(contentType, contenttype.XMLUTF8) w.Header().Set(contentType, contenttype.XMLUTF8)
var buf bytes.Buffer pipeReader, pipeWriter := io.Pipe()
buf.WriteString(xml.Header) go func() {
buf.WriteString(`<?xml-stylesheet type="text/xsl" href="`) mw := a.min.Writer(contenttype.XML, pipeWriter)
buf.WriteString(a.assetFileName("sitemap.xsl")) _, _ = io.WriteString(mw, xml.Header)
buf.WriteString(`" ?>`) _, _ = io.WriteString(mw, `<?xml-stylesheet type="text/xsl" href="`)
_ = xml.NewEncoder(&buf).Encode(sm) _, _ = io.WriteString(mw, a.assetFileName("sitemap.xsl"))
_ = a.min.Minify(contenttype.XML, w, &buf) _, _ = io.WriteString(mw, `" ?>`)
writeErr := xml.NewEncoder(mw).Encode(sm)
_ = mw.Close()
_ = pipeWriter.CloseWithError(writeErr)
}()
_, copyErr := io.Copy(w, pipeReader)
_ = pipeReader.CloseWithError(copyErr)
} }
const sitemapDatePathsSql = ` const sitemapDatePathsSql = `