From 856b504877fd84a7b2631026894e22e73aca89b4 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Else Date: Wed, 23 Feb 2022 21:33:02 +0100 Subject: [PATCH] More pooled buffers, benchmarks and optional pprof server --- comments.go | 4 +++- config.go | 8 +++++++- editor.go | 35 ++++++++++++++++++----------------- main.go | 36 ++++++++++++++++++++++++++++++++++++ markdown.go | 33 +++++++++++++++++---------------- markdown_test.go | 14 +++++++++++++- notifications.go | 5 +++-- postsDb.go | 18 +++++++++++------- templateAssets.go | 20 ++++++++++++-------- tts.go | 35 +++++++++++++++++------------------ webmention.go | 4 +++- 11 files changed, 140 insertions(+), 72 deletions(-) diff --git a/comments.go b/comments.go index 73e74b2..344a024 100644 --- a/comments.go +++ b/comments.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/go-chi/chi/v5" + "go.goblog.app/app/pkgs/bufferpool" ) const commentPath = "/comment" @@ -102,7 +103,8 @@ type commentsRequestConfig struct { } func buildCommentsQuery(config *commentsRequestConfig) (query string, args []interface{}) { - var queryBuilder strings.Builder + queryBuilder := bufferpool.Get() + defer bufferpool.Put(queryBuilder) queryBuilder.WriteString("select id, target, name, website, comment from comments order by id desc") if config.limit != 0 || config.offset != 0 { queryBuilder.WriteString(" limit @limit offset @offset") diff --git a/config.go b/config.go index f99bd0a..39a5d84 100644 --- a/config.go +++ b/config.go @@ -26,9 +26,10 @@ type config struct { PrivateMode *configPrivateMode `mapstructure:"privateMode"` IndexNow *configIndexNow `mapstructure:"indexNow"` EasterEgg *configEasterEgg `mapstructure:"easterEgg"` - Debug bool `mapstructure:"debug"` MapTiles *configMapTiles `mapstructure:"mapTiles"` TTS *configTTS `mapstructure:"tts"` + Pprof *configPprof `mapstructure:"pprof"` + Debug bool `mapstructure:"debug"` initialized bool } @@ -302,6 +303,11 @@ type configTTS struct { GoogleAPIKey string `mapstructure:"googleApiKey"` } +type configPprof struct { + Enabled bool `mapstructure:"enabled"` + Address string `mapstructure:"address"` +} + func (a *goBlog) loadConfigFile(file string) error { // Use viper to load the config file v := viper.New() diff --git a/editor.go b/editor.go index f053f5f..b8625ed 100644 --- a/editor.go +++ b/editor.go @@ -8,7 +8,6 @@ import ( "net/http" "net/http/httptest" "net/url" - "strings" "time" "go.goblog.app/app/pkgs/bufferpool" @@ -93,21 +92,22 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) { }, }) case "updatepost": - pipeReader, pipeWriter := io.Pipe() - defer pipeReader.Close() - go func() { - writeErr := json.NewEncoder(pipeWriter).Encode(map[string]interface{}{ - "action": actionUpdate, - "url": r.FormValue("url"), - "replace": map[string][]string{ - "content": { - r.FormValue("content"), - }, + buf := bufferpool.Get() + defer bufferpool.Put(buf) + err := json.NewEncoder(buf).Encode(map[string]interface{}{ + "action": actionUpdate, + "url": r.FormValue("url"), + "replace": map[string][]string{ + "content": { + r.FormValue("content"), }, - }) - _ = pipeWriter.CloseWithError(writeErr) - }() - req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, "", pipeReader) + }, + }) + if err != nil { + a.serveError(w, r, err.Error(), http.StatusInternalServerError) + return + } + req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, "", buf) if err != nil { a.serveError(w, r, err.Error(), http.StatusInternalServerError) return @@ -177,9 +177,10 @@ func (a *goBlog) editorMicropubPost(w http.ResponseWriter, r *http.Request, medi } func (a *goBlog) editorPostTemplate(blog string, bc *configBlog) string { - var builder strings.Builder + builder := bufferpool.Get() + defer bufferpool.Put(builder) marsh := func(param string, i interface{}) { - _ = yaml.NewEncoder(&builder).Encode(map[string]interface{}{ + _ = yaml.NewEncoder(builder).Encode(map[string]interface{}{ param: i, }) } diff --git a/main.go b/main.go index 6dde00e..2a946bd 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,9 @@ package main import ( "flag" "log" + "net" + "net/http" + netpprof "net/http/pprof" "os" "runtime" "runtime/pprof" @@ -90,6 +93,39 @@ func main() { return } + // Start pprof server + if pprofCfg := app.cfg.Pprof; pprofCfg != nil && pprofCfg.Enabled { + go func() { + // Build handler + pprofHandler := http.NewServeMux() + pprofHandler.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) { + http.Redirect(rw, r, "/debug/pprof/", http.StatusFound) + }) + pprofHandler.HandleFunc("/debug/pprof/", netpprof.Index) + pprofHandler.HandleFunc("/debug/pprof/{action}", netpprof.Index) + pprofHandler.HandleFunc("/debug/pprof/cmdline", netpprof.Cmdline) + pprofHandler.HandleFunc("/debug/pprof/profile", netpprof.Profile) + pprofHandler.HandleFunc("/debug/pprof/symbol", netpprof.Symbol) + pprofHandler.HandleFunc("/debug/pprof/trace", netpprof.Trace) + // Build server and listener + pprofServer := &http.Server{ + Addr: defaultIfEmpty(pprofCfg.Address, "localhost:0"), + Handler: pprofHandler, + } + listener, err := net.Listen("tcp", pprofServer.Addr) + if err != nil { + log.Fatalln("Failed to start pprof server:", err.Error()) + return + } + log.Println("Pprof server listening on", listener.Addr().String()) + // Start server + if err := pprofServer.Serve(listener); err != nil { + log.Fatalln("Failed to start pprof server:", err.Error()) + return + } + }() + } + // Execute pre-start hooks app.preStartHooks() diff --git a/markdown.go b/markdown.go index d40df98..da4aca4 100644 --- a/markdown.go +++ b/markdown.go @@ -15,6 +15,7 @@ import ( "github.com/yuin/goldmark/renderer" "github.com/yuin/goldmark/renderer/html" "github.com/yuin/goldmark/util" + "go.goblog.app/app/pkgs/bufferpool" ) func (a *goBlog) initMarkdown() { @@ -88,14 +89,14 @@ func (a *goBlog) renderText(s string) string { if s == "" { return "" } - pipeReader, pipeWriter := io.Pipe() - go func() { - writeErr := a.renderMarkdownToWriter(pipeWriter, s, false) - _ = pipeWriter.CloseWithError(writeErr) - }() - text, readErr := htmlTextFromReader(pipeReader) - _ = pipeReader.CloseWithError(readErr) - if readErr != nil { + buf := bufferpool.Get() + defer bufferpool.Put(buf) + err := a.renderMarkdownToWriter(buf, s, false) + if err != nil { + return "" + } + text, err := htmlTextFromReader(buf) + if err != nil { return "" } return text @@ -105,14 +106,14 @@ func (a *goBlog) renderMdTitle(s string) string { if s == "" { return "" } - pipeReader, pipeWriter := io.Pipe() - go func() { - writeErr := a.titleMd.Convert([]byte(s), pipeWriter) - _ = pipeWriter.CloseWithError(writeErr) - }() - text, readErr := htmlTextFromReader(pipeReader) - _ = pipeReader.CloseWithError(readErr) - if readErr != nil { + buf := bufferpool.Get() + defer bufferpool.Put(buf) + err := a.titleMd.Convert([]byte(s), buf) + if err != nil { + return "" + } + text, err := htmlTextFromReader(buf) + if err != nil { return "" } return text diff --git a/markdown_test.go b/markdown_test.go index 6739297..3846c2b 100644 --- a/markdown_test.go +++ b/markdown_test.go @@ -108,7 +108,7 @@ func Benchmark_markdown(b *testing.B) { app.initMarkdown() - b.Run("Benchmark Markdown Rendering", func(b *testing.B) { + b.Run("Markdown Rendering", func(b *testing.B) { for i := 0; i < b.N; i++ { _, err := app.renderMarkdown(mdExp, true) if err != nil { @@ -116,4 +116,16 @@ func Benchmark_markdown(b *testing.B) { } } }) + + b.Run("Title Rendering", func(b *testing.B) { + for i := 0; i < b.N; i++ { + app.renderMdTitle("**Test**") + } + }) + + b.Run("Text Rendering", func(b *testing.B) { + for i := 0; i < b.N; i++ { + app.renderText("**Test**") + } + }) } diff --git a/notifications.go b/notifications.go index a60f3f3..7436cb8 100644 --- a/notifications.go +++ b/notifications.go @@ -7,11 +7,11 @@ import ( "net/http" "reflect" "strconv" - "strings" "time" "github.com/go-chi/chi/v5" "github.com/vcraescu/go-paginator" + "go.goblog.app/app/pkgs/bufferpool" ) const notificationsPath = "/notifications" @@ -57,7 +57,8 @@ type notificationsRequestConfig struct { } func buildNotificationsQuery(config *notificationsRequestConfig) (query string, args []interface{}) { - var queryBuilder strings.Builder + queryBuilder := bufferpool.Get() + defer bufferpool.Put(queryBuilder) queryBuilder.WriteString("select id, time, text from notifications order by id desc") if config.limit != 0 || config.offset != 0 { queryBuilder.WriteString(" limit @limit offset @offset") diff --git a/postsDb.go b/postsDb.go index 17f0434..a30094e 100644 --- a/postsDb.go +++ b/postsDb.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "database/sql" "errors" "fmt" @@ -99,8 +98,9 @@ func (a *goBlog) checkPost(p *post) (err error) { if err != nil { return errors.New("failed to parse location template") } - var pathBuffer bytes.Buffer - err = pathTmpl.Execute(&pathBuffer, map[string]interface{}{ + pathBuffer := bufferpool.Get() + defer bufferpool.Put(pathBuffer) + err = pathTmpl.Execute(pathBuffer, map[string]interface{}{ "BlogPath": a.getRelativePath(p.Blog, ""), "Year": published.Year(), "Month": int(published.Month()), @@ -171,7 +171,8 @@ func (db *database) savePost(p *post, o *postCreationOptions) error { db.pcm.Lock() defer db.pcm.Unlock() // Build SQL - var sqlBuilder strings.Builder + sqlBuilder := bufferpool.Get() + defer bufferpool.Put(sqlBuilder) var sqlArgs = []interface{}{dbNoCache} // Start transaction sqlBuilder.WriteString("begin;") @@ -294,7 +295,8 @@ func (db *database) replacePostParam(path, param string, values []string) error db.pcm.Lock() defer db.pcm.Unlock() // Build SQL - var sqlBuilder strings.Builder + sqlBuilder := bufferpool.Get() + defer bufferpool.Put(sqlBuilder) var sqlArgs = []interface{}{dbNoCache} // Start transaction sqlBuilder.WriteString("begin;") @@ -343,7 +345,8 @@ type postsRequestConfig struct { } func buildPostsQuery(c *postsRequestConfig, selection string) (query string, args []interface{}) { - var queryBuilder strings.Builder + queryBuilder := bufferpool.Get() + defer bufferpool.Put(queryBuilder) // Selection queryBuilder.WriteString("select ") queryBuilder.WriteString(selection) @@ -459,7 +462,8 @@ func (d *database) loadPostParameters(posts []*post, parameters ...string) (err } // Build query sqlArgs := make([]interface{}, 0) - var queryBuilder strings.Builder + queryBuilder := bufferpool.Get() + defer bufferpool.Put(queryBuilder) queryBuilder.WriteString("select path, parameter, value from post_parameters where") // Paths queryBuilder.WriteString(" path in (") diff --git a/templateAssets.go b/templateAssets.go index e236aad..f7c9293 100644 --- a/templateAssets.go +++ b/templateAssets.go @@ -12,6 +12,7 @@ import ( "strings" chromahtml "github.com/alecthomas/chroma/formatters/html" + "go.goblog.app/app/pkgs/bufferpool" "go.goblog.app/app/pkgs/contenttype" ) @@ -115,12 +116,15 @@ func (a *goBlog) initChromaCSS() error { return err } // Generate and minify CSS - pipeReader, pipeWriter := io.Pipe() - go func() { - writeErr := chromahtml.New(chromahtml.ClassPrefix("c-")).WriteCSS(pipeWriter, chromaStyle) - _ = pipeWriter.CloseWithError(writeErr) - }() - readErr := a.compileAsset(chromaPath, pipeReader) - _ = pipeReader.CloseWithError(readErr) - return readErr + buf := bufferpool.Get() + defer bufferpool.Put(buf) + err = chromahtml.New(chromahtml.ClassPrefix("c-")).WriteCSS(buf, chromaStyle) + if err != nil { + return err + } + err = a.compileAsset(chromaPath, buf) + if err != nil { + return err + } + return nil } diff --git a/tts.go b/tts.go index c67e839..0e79379 100644 --- a/tts.go +++ b/tts.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "context" "crypto/sha256" "encoding/base64" @@ -73,31 +72,29 @@ func (a *goBlog) createPostTTSAudio(p *post) error { parts = append(parts, strings.Split(htmlText(a.postHtml(p, false)), "\n\n")...) // Create TTS audio for each part - partsBuffers := make([]io.Reader, len(parts)) - var errs []error - var lock sync.Mutex + partWriters := make([]io.Writer, len(parts)) + partReaders := make([]io.Reader, len(parts)) + for i := range parts { + buf := bufferpool.Get() + defer bufferpool.Put(buf) + partWriters[i] = buf + partReaders[i] = buf + } + errs := make([]error, len(parts)) var wg sync.WaitGroup for i, part := range parts { // Increase wait group wg.Add(1) go func(i int, part string) { + defer wg.Done() // Build SSML ssml := "" + html.EscapeString(part) + "" // Create TTS audio - var audioBuffer bytes.Buffer - err := a.createTTSAudio(lang, ssml, &audioBuffer) + err := a.createTTSAudio(lang, ssml, partWriters[i]) if err != nil { - lock.Lock() - errs = append(errs, err) - lock.Unlock() + errs[i] = err return } - // Append buffer to partsBuffers - lock.Lock() - partsBuffers[i] = &audioBuffer - lock.Unlock() - // Decrease wait group - wg.Done() }(i, part) } @@ -105,15 +102,17 @@ func (a *goBlog) createPostTTSAudio(p *post) error { wg.Wait() // Check if any errors occurred - if len(errs) > 0 { - return errs[0] + for _, err := range errs { + if err != nil { + return err + } } // Merge partsBuffers into final buffer final := bufferpool.Get() defer bufferpool.Put(final) hash := sha256.New() - if err := mp3merge.MergeMP3(io.MultiWriter(final, hash), partsBuffers...); err != nil { + if err := mp3merge.MergeMP3(io.MultiWriter(final, hash), partReaders...); err != nil { return err } diff --git a/webmention.go b/webmention.go index 6f56235..3f7574e 100644 --- a/webmention.go +++ b/webmention.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "go.goblog.app/app/pkgs/bufferpool" "go.goblog.app/app/pkgs/contenttype" ) @@ -226,7 +227,8 @@ type webmentionsRequestConfig struct { } func buildWebmentionsQuery(config *webmentionsRequestConfig) (query string, args []interface{}) { - var queryBuilder strings.Builder + queryBuilder := bufferpool.Get() + defer bufferpool.Put(queryBuilder) queryBuilder.WriteString("select id, source, target, url, created, title, content, author, status from webmentions ") if config != nil { queryBuilder.WriteString("where 1")