More pooled buffers, benchmarks and optional pprof server

This commit is contained in:
Jan-Lukas Else 2022-02-23 21:33:02 +01:00
parent 68b2d604c3
commit 856b504877
11 changed files with 140 additions and 72 deletions

View File

@ -9,6 +9,7 @@ import (
"strings" "strings"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"go.goblog.app/app/pkgs/bufferpool"
) )
const commentPath = "/comment" const commentPath = "/comment"
@ -102,7 +103,8 @@ type commentsRequestConfig struct {
} }
func buildCommentsQuery(config *commentsRequestConfig) (query string, args []interface{}) { func buildCommentsQuery(config *commentsRequestConfig) (query string, args []interface{}) {
var queryBuilder strings.Builder queryBuilder := bufferpool.Get()
defer bufferpool.Put(queryBuilder)
queryBuilder.WriteString("select id, target, name, website, comment from comments order by id desc") queryBuilder.WriteString("select id, target, name, website, comment from comments order by id desc")
if config.limit != 0 || config.offset != 0 { if config.limit != 0 || config.offset != 0 {
queryBuilder.WriteString(" limit @limit offset @offset") queryBuilder.WriteString(" limit @limit offset @offset")

View File

@ -26,9 +26,10 @@ type config struct {
PrivateMode *configPrivateMode `mapstructure:"privateMode"` PrivateMode *configPrivateMode `mapstructure:"privateMode"`
IndexNow *configIndexNow `mapstructure:"indexNow"` IndexNow *configIndexNow `mapstructure:"indexNow"`
EasterEgg *configEasterEgg `mapstructure:"easterEgg"` EasterEgg *configEasterEgg `mapstructure:"easterEgg"`
Debug bool `mapstructure:"debug"`
MapTiles *configMapTiles `mapstructure:"mapTiles"` MapTiles *configMapTiles `mapstructure:"mapTiles"`
TTS *configTTS `mapstructure:"tts"` TTS *configTTS `mapstructure:"tts"`
Pprof *configPprof `mapstructure:"pprof"`
Debug bool `mapstructure:"debug"`
initialized bool initialized bool
} }
@ -302,6 +303,11 @@ type configTTS struct {
GoogleAPIKey string `mapstructure:"googleApiKey"` GoogleAPIKey string `mapstructure:"googleApiKey"`
} }
type configPprof struct {
Enabled bool `mapstructure:"enabled"`
Address string `mapstructure:"address"`
}
func (a *goBlog) loadConfigFile(file string) error { func (a *goBlog) loadConfigFile(file string) error {
// Use viper to load the config file // Use viper to load the config file
v := viper.New() v := viper.New()

View File

@ -8,7 +8,6 @@ import (
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/url" "net/url"
"strings"
"time" "time"
"go.goblog.app/app/pkgs/bufferpool" "go.goblog.app/app/pkgs/bufferpool"
@ -93,21 +92,22 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) {
}, },
}) })
case "updatepost": case "updatepost":
pipeReader, pipeWriter := io.Pipe() buf := bufferpool.Get()
defer pipeReader.Close() defer bufferpool.Put(buf)
go func() { err := json.NewEncoder(buf).Encode(map[string]interface{}{
writeErr := json.NewEncoder(pipeWriter).Encode(map[string]interface{}{ "action": actionUpdate,
"action": actionUpdate, "url": r.FormValue("url"),
"url": r.FormValue("url"), "replace": map[string][]string{
"replace": map[string][]string{ "content": {
"content": { r.FormValue("content"),
r.FormValue("content"),
},
}, },
}) },
_ = pipeWriter.CloseWithError(writeErr) })
}() if err != nil {
req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, "", pipeReader) a.serveError(w, r, err.Error(), http.StatusInternalServerError)
return
}
req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, "", buf)
if err != nil { if err != nil {
a.serveError(w, r, err.Error(), http.StatusInternalServerError) a.serveError(w, r, err.Error(), http.StatusInternalServerError)
return return
@ -177,9 +177,10 @@ func (a *goBlog) editorMicropubPost(w http.ResponseWriter, r *http.Request, medi
} }
func (a *goBlog) editorPostTemplate(blog string, bc *configBlog) string { func (a *goBlog) editorPostTemplate(blog string, bc *configBlog) string {
var builder strings.Builder builder := bufferpool.Get()
defer bufferpool.Put(builder)
marsh := func(param string, i interface{}) { marsh := func(param string, i interface{}) {
_ = yaml.NewEncoder(&builder).Encode(map[string]interface{}{ _ = yaml.NewEncoder(builder).Encode(map[string]interface{}{
param: i, param: i,
}) })
} }

36
main.go
View File

@ -3,6 +3,9 @@ package main
import ( import (
"flag" "flag"
"log" "log"
"net"
"net/http"
netpprof "net/http/pprof"
"os" "os"
"runtime" "runtime"
"runtime/pprof" "runtime/pprof"
@ -90,6 +93,39 @@ func main() {
return return
} }
// Start pprof server
if pprofCfg := app.cfg.Pprof; pprofCfg != nil && pprofCfg.Enabled {
go func() {
// Build handler
pprofHandler := http.NewServeMux()
pprofHandler.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
http.Redirect(rw, r, "/debug/pprof/", http.StatusFound)
})
pprofHandler.HandleFunc("/debug/pprof/", netpprof.Index)
pprofHandler.HandleFunc("/debug/pprof/{action}", netpprof.Index)
pprofHandler.HandleFunc("/debug/pprof/cmdline", netpprof.Cmdline)
pprofHandler.HandleFunc("/debug/pprof/profile", netpprof.Profile)
pprofHandler.HandleFunc("/debug/pprof/symbol", netpprof.Symbol)
pprofHandler.HandleFunc("/debug/pprof/trace", netpprof.Trace)
// Build server and listener
pprofServer := &http.Server{
Addr: defaultIfEmpty(pprofCfg.Address, "localhost:0"),
Handler: pprofHandler,
}
listener, err := net.Listen("tcp", pprofServer.Addr)
if err != nil {
log.Fatalln("Failed to start pprof server:", err.Error())
return
}
log.Println("Pprof server listening on", listener.Addr().String())
// Start server
if err := pprofServer.Serve(listener); err != nil {
log.Fatalln("Failed to start pprof server:", err.Error())
return
}
}()
}
// Execute pre-start hooks // Execute pre-start hooks
app.preStartHooks() app.preStartHooks()

View File

@ -15,6 +15,7 @@ import (
"github.com/yuin/goldmark/renderer" "github.com/yuin/goldmark/renderer"
"github.com/yuin/goldmark/renderer/html" "github.com/yuin/goldmark/renderer/html"
"github.com/yuin/goldmark/util" "github.com/yuin/goldmark/util"
"go.goblog.app/app/pkgs/bufferpool"
) )
func (a *goBlog) initMarkdown() { func (a *goBlog) initMarkdown() {
@ -88,14 +89,14 @@ func (a *goBlog) renderText(s string) string {
if s == "" { if s == "" {
return "" return ""
} }
pipeReader, pipeWriter := io.Pipe() buf := bufferpool.Get()
go func() { defer bufferpool.Put(buf)
writeErr := a.renderMarkdownToWriter(pipeWriter, s, false) err := a.renderMarkdownToWriter(buf, s, false)
_ = pipeWriter.CloseWithError(writeErr) if err != nil {
}() return ""
text, readErr := htmlTextFromReader(pipeReader) }
_ = pipeReader.CloseWithError(readErr) text, err := htmlTextFromReader(buf)
if readErr != nil { if err != nil {
return "" return ""
} }
return text return text
@ -105,14 +106,14 @@ func (a *goBlog) renderMdTitle(s string) string {
if s == "" { if s == "" {
return "" return ""
} }
pipeReader, pipeWriter := io.Pipe() buf := bufferpool.Get()
go func() { defer bufferpool.Put(buf)
writeErr := a.titleMd.Convert([]byte(s), pipeWriter) err := a.titleMd.Convert([]byte(s), buf)
_ = pipeWriter.CloseWithError(writeErr) if err != nil {
}() return ""
text, readErr := htmlTextFromReader(pipeReader) }
_ = pipeReader.CloseWithError(readErr) text, err := htmlTextFromReader(buf)
if readErr != nil { if err != nil {
return "" return ""
} }
return text return text

View File

@ -108,7 +108,7 @@ func Benchmark_markdown(b *testing.B) {
app.initMarkdown() app.initMarkdown()
b.Run("Benchmark Markdown Rendering", func(b *testing.B) { b.Run("Markdown Rendering", func(b *testing.B) {
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
_, err := app.renderMarkdown(mdExp, true) _, err := app.renderMarkdown(mdExp, true)
if err != nil { if err != nil {
@ -116,4 +116,16 @@ func Benchmark_markdown(b *testing.B) {
} }
} }
}) })
b.Run("Title Rendering", func(b *testing.B) {
for i := 0; i < b.N; i++ {
app.renderMdTitle("**Test**")
}
})
b.Run("Text Rendering", func(b *testing.B) {
for i := 0; i < b.N; i++ {
app.renderText("**Test**")
}
})
} }

View File

@ -7,11 +7,11 @@ import (
"net/http" "net/http"
"reflect" "reflect"
"strconv" "strconv"
"strings"
"time" "time"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/vcraescu/go-paginator" "github.com/vcraescu/go-paginator"
"go.goblog.app/app/pkgs/bufferpool"
) )
const notificationsPath = "/notifications" const notificationsPath = "/notifications"
@ -57,7 +57,8 @@ type notificationsRequestConfig struct {
} }
func buildNotificationsQuery(config *notificationsRequestConfig) (query string, args []interface{}) { func buildNotificationsQuery(config *notificationsRequestConfig) (query string, args []interface{}) {
var queryBuilder strings.Builder queryBuilder := bufferpool.Get()
defer bufferpool.Put(queryBuilder)
queryBuilder.WriteString("select id, time, text from notifications order by id desc") queryBuilder.WriteString("select id, time, text from notifications order by id desc")
if config.limit != 0 || config.offset != 0 { if config.limit != 0 || config.offset != 0 {
queryBuilder.WriteString(" limit @limit offset @offset") queryBuilder.WriteString(" limit @limit offset @offset")

View File

@ -1,7 +1,6 @@
package main package main
import ( import (
"bytes"
"database/sql" "database/sql"
"errors" "errors"
"fmt" "fmt"
@ -99,8 +98,9 @@ func (a *goBlog) checkPost(p *post) (err error) {
if err != nil { if err != nil {
return errors.New("failed to parse location template") return errors.New("failed to parse location template")
} }
var pathBuffer bytes.Buffer pathBuffer := bufferpool.Get()
err = pathTmpl.Execute(&pathBuffer, map[string]interface{}{ defer bufferpool.Put(pathBuffer)
err = pathTmpl.Execute(pathBuffer, map[string]interface{}{
"BlogPath": a.getRelativePath(p.Blog, ""), "BlogPath": a.getRelativePath(p.Blog, ""),
"Year": published.Year(), "Year": published.Year(),
"Month": int(published.Month()), "Month": int(published.Month()),
@ -171,7 +171,8 @@ func (db *database) savePost(p *post, o *postCreationOptions) error {
db.pcm.Lock() db.pcm.Lock()
defer db.pcm.Unlock() defer db.pcm.Unlock()
// Build SQL // Build SQL
var sqlBuilder strings.Builder sqlBuilder := bufferpool.Get()
defer bufferpool.Put(sqlBuilder)
var sqlArgs = []interface{}{dbNoCache} var sqlArgs = []interface{}{dbNoCache}
// Start transaction // Start transaction
sqlBuilder.WriteString("begin;") sqlBuilder.WriteString("begin;")
@ -294,7 +295,8 @@ func (db *database) replacePostParam(path, param string, values []string) error
db.pcm.Lock() db.pcm.Lock()
defer db.pcm.Unlock() defer db.pcm.Unlock()
// Build SQL // Build SQL
var sqlBuilder strings.Builder sqlBuilder := bufferpool.Get()
defer bufferpool.Put(sqlBuilder)
var sqlArgs = []interface{}{dbNoCache} var sqlArgs = []interface{}{dbNoCache}
// Start transaction // Start transaction
sqlBuilder.WriteString("begin;") sqlBuilder.WriteString("begin;")
@ -343,7 +345,8 @@ type postsRequestConfig struct {
} }
func buildPostsQuery(c *postsRequestConfig, selection string) (query string, args []interface{}) { func buildPostsQuery(c *postsRequestConfig, selection string) (query string, args []interface{}) {
var queryBuilder strings.Builder queryBuilder := bufferpool.Get()
defer bufferpool.Put(queryBuilder)
// Selection // Selection
queryBuilder.WriteString("select ") queryBuilder.WriteString("select ")
queryBuilder.WriteString(selection) queryBuilder.WriteString(selection)
@ -459,7 +462,8 @@ func (d *database) loadPostParameters(posts []*post, parameters ...string) (err
} }
// Build query // Build query
sqlArgs := make([]interface{}, 0) sqlArgs := make([]interface{}, 0)
var queryBuilder strings.Builder queryBuilder := bufferpool.Get()
defer bufferpool.Put(queryBuilder)
queryBuilder.WriteString("select path, parameter, value from post_parameters where") queryBuilder.WriteString("select path, parameter, value from post_parameters where")
// Paths // Paths
queryBuilder.WriteString(" path in (") queryBuilder.WriteString(" path in (")

View File

@ -12,6 +12,7 @@ import (
"strings" "strings"
chromahtml "github.com/alecthomas/chroma/formatters/html" chromahtml "github.com/alecthomas/chroma/formatters/html"
"go.goblog.app/app/pkgs/bufferpool"
"go.goblog.app/app/pkgs/contenttype" "go.goblog.app/app/pkgs/contenttype"
) )
@ -115,12 +116,15 @@ func (a *goBlog) initChromaCSS() error {
return err return err
} }
// Generate and minify CSS // Generate and minify CSS
pipeReader, pipeWriter := io.Pipe() buf := bufferpool.Get()
go func() { defer bufferpool.Put(buf)
writeErr := chromahtml.New(chromahtml.ClassPrefix("c-")).WriteCSS(pipeWriter, chromaStyle) err = chromahtml.New(chromahtml.ClassPrefix("c-")).WriteCSS(buf, chromaStyle)
_ = pipeWriter.CloseWithError(writeErr) if err != nil {
}() return err
readErr := a.compileAsset(chromaPath, pipeReader) }
_ = pipeReader.CloseWithError(readErr) err = a.compileAsset(chromaPath, buf)
return readErr if err != nil {
return err
}
return nil
} }

35
tts.go
View File

@ -1,7 +1,6 @@
package main package main
import ( import (
"bytes"
"context" "context"
"crypto/sha256" "crypto/sha256"
"encoding/base64" "encoding/base64"
@ -73,31 +72,29 @@ func (a *goBlog) createPostTTSAudio(p *post) error {
parts = append(parts, strings.Split(htmlText(a.postHtml(p, false)), "\n\n")...) parts = append(parts, strings.Split(htmlText(a.postHtml(p, false)), "\n\n")...)
// Create TTS audio for each part // Create TTS audio for each part
partsBuffers := make([]io.Reader, len(parts)) partWriters := make([]io.Writer, len(parts))
var errs []error partReaders := make([]io.Reader, len(parts))
var lock sync.Mutex for i := range parts {
buf := bufferpool.Get()
defer bufferpool.Put(buf)
partWriters[i] = buf
partReaders[i] = buf
}
errs := make([]error, len(parts))
var wg sync.WaitGroup var wg sync.WaitGroup
for i, part := range parts { for i, part := range parts {
// Increase wait group // Increase wait group
wg.Add(1) wg.Add(1)
go func(i int, part string) { go func(i int, part string) {
defer wg.Done()
// Build SSML // Build SSML
ssml := "<speak>" + html.EscapeString(part) + "<break time=\"500ms\"/></speak>" ssml := "<speak>" + html.EscapeString(part) + "<break time=\"500ms\"/></speak>"
// Create TTS audio // Create TTS audio
var audioBuffer bytes.Buffer err := a.createTTSAudio(lang, ssml, partWriters[i])
err := a.createTTSAudio(lang, ssml, &audioBuffer)
if err != nil { if err != nil {
lock.Lock() errs[i] = err
errs = append(errs, err)
lock.Unlock()
return return
} }
// Append buffer to partsBuffers
lock.Lock()
partsBuffers[i] = &audioBuffer
lock.Unlock()
// Decrease wait group
wg.Done()
}(i, part) }(i, part)
} }
@ -105,15 +102,17 @@ func (a *goBlog) createPostTTSAudio(p *post) error {
wg.Wait() wg.Wait()
// Check if any errors occurred // Check if any errors occurred
if len(errs) > 0 { for _, err := range errs {
return errs[0] if err != nil {
return err
}
} }
// Merge partsBuffers into final buffer // Merge partsBuffers into final buffer
final := bufferpool.Get() final := bufferpool.Get()
defer bufferpool.Put(final) defer bufferpool.Put(final)
hash := sha256.New() hash := sha256.New()
if err := mp3merge.MergeMP3(io.MultiWriter(final, hash), partsBuffers...); err != nil { if err := mp3merge.MergeMP3(io.MultiWriter(final, hash), partReaders...); err != nil {
return err return err
} }

View File

@ -8,6 +8,7 @@ import (
"strings" "strings"
"time" "time"
"go.goblog.app/app/pkgs/bufferpool"
"go.goblog.app/app/pkgs/contenttype" "go.goblog.app/app/pkgs/contenttype"
) )
@ -226,7 +227,8 @@ type webmentionsRequestConfig struct {
} }
func buildWebmentionsQuery(config *webmentionsRequestConfig) (query string, args []interface{}) { func buildWebmentionsQuery(config *webmentionsRequestConfig) (query string, args []interface{}) {
var queryBuilder strings.Builder queryBuilder := bufferpool.Get()
defer bufferpool.Put(queryBuilder)
queryBuilder.WriteString("select id, source, target, url, created, title, content, author, status from webmentions ") queryBuilder.WriteString("select id, source, target, url, created, title, content, author, status from webmentions ")
if config != nil { if config != nil {
queryBuilder.WriteString("where 1") queryBuilder.WriteString("where 1")