jlelse
/
GoBlog
mirror of https://github.com/jlelse/GoBlog
1
Fork 0
Code Packages Releases Activity

More pooled buffers, benchmarks and optional pprof server

This commit is contained in:
Jan-Lukas Else 2022-02-23 21:33:02 +01:00
parent 68b2d604c3
commit 856b504877
11 changed files with 140 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
comments.go
View File

@ -9,6 +9,7 @@ import (
"strings"
"github.com/go-chi/chi/v5"
"go.goblog.app/app/pkgs/bufferpool"
)
const commentPath = "/comment"
@ -102,7 +103,8 @@ type commentsRequestConfig struct {
}
func buildCommentsQuery(config *commentsRequestConfig) (query string, args []interface{}) {
var queryBuilder strings.Builder
queryBuilder := bufferpool.Get()
defer bufferpool.Put(queryBuilder)
queryBuilder.WriteString("select id, target, name, website, comment from comments order by id desc")
if config.limit != 0 || config.offset != 0 {
queryBuilder.WriteString(" limit @limit offset @offset")

8
config.go
View File

@ -26,9 +26,10 @@ type config struct {
PrivateMode *configPrivateMode `mapstructure:"privateMode"`
IndexNow *configIndexNow `mapstructure:"indexNow"`
EasterEgg *configEasterEgg `mapstructure:"easterEgg"`
Debug bool `mapstructure:"debug"`
MapTiles *configMapTiles `mapstructure:"mapTiles"`
TTS *configTTS `mapstructure:"tts"`
Pprof *configPprof `mapstructure:"pprof"`
Debug bool `mapstructure:"debug"`
initialized bool
}
@ -302,6 +303,11 @@ type configTTS struct {
GoogleAPIKey string `mapstructure:"googleApiKey"`
}
type configPprof struct {
Enabled bool `mapstructure:"enabled"`
Address string `mapstructure:"address"`
}
func (a *goBlog) loadConfigFile(file string) error {
// Use viper to load the config file
v := viper.New()

21
editor.go
View File

@ -8,7 +8,6 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"strings"
"time"
"go.goblog.app/app/pkgs/bufferpool"
@ -93,10 +92,9 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) {
},
})
case "updatepost":
pipeReader, pipeWriter := io.Pipe()
defer pipeReader.Close()
go func() {
writeErr := json.NewEncoder(pipeWriter).Encode(map[string]interface{}{
buf := bufferpool.Get()
defer bufferpool.Put(buf)
err := json.NewEncoder(buf).Encode(map[string]interface{}{
"action": actionUpdate,
"url": r.FormValue("url"),
"replace": map[string][]string{
@ -105,9 +103,11 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) {
},
},
})
_ = pipeWriter.CloseWithError(writeErr)
}()
req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, "", pipeReader)
if err != nil {
a.serveError(w, r, err.Error(), http.StatusInternalServerError)
return
}
req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, "", buf)
if err != nil {
a.serveError(w, r, err.Error(), http.StatusInternalServerError)
return
@ -177,9 +177,10 @@ func (a *goBlog) editorMicropubPost(w http.ResponseWriter, r *http.Request, medi
}
func (a *goBlog) editorPostTemplate(blog string, bc *configBlog) string {
var builder strings.Builder
builder := bufferpool.Get()
defer bufferpool.Put(builder)
marsh := func(param string, i interface{}) {
_ = yaml.NewEncoder(&builder).Encode(map[string]interface{}{
_ = yaml.NewEncoder(builder).Encode(map[string]interface{}{
param: i,
})
}

36
main.go
View File

@ -3,6 +3,9 @@ package main
import (
"flag"
"log"
"net"
"net/http"
netpprof "net/http/pprof"
"os"
"runtime"
"runtime/pprof"
@ -90,6 +93,39 @@ func main() {
return
}
// Start pprof server
if pprofCfg := app.cfg.Pprof; pprofCfg != nil && pprofCfg.Enabled {
go func() {
// Build handler
pprofHandler := http.NewServeMux()
pprofHandler.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
http.Redirect(rw, r, "/debug/pprof/", http.StatusFound)
})
pprofHandler.HandleFunc("/debug/pprof/", netpprof.Index)
pprofHandler.HandleFunc("/debug/pprof/{action}", netpprof.Index)
pprofHandler.HandleFunc("/debug/pprof/cmdline", netpprof.Cmdline)
pprofHandler.HandleFunc("/debug/pprof/profile", netpprof.Profile)
pprofHandler.HandleFunc("/debug/pprof/symbol", netpprof.Symbol)
pprofHandler.HandleFunc("/debug/pprof/trace", netpprof.Trace)
// Build server and listener
pprofServer := &http.Server{
Addr: defaultIfEmpty(pprofCfg.Address, "localhost:0"),
Handler: pprofHandler,
}
listener, err := net.Listen("tcp", pprofServer.Addr)
if err != nil {
log.Fatalln("Failed to start pprof server:", err.Error())
return
}
log.Println("Pprof server listening on", listener.Addr().String())
// Start server
if err := pprofServer.Serve(listener); err != nil {
log.Fatalln("Failed to start pprof server:", err.Error())
return
}
}()
}
// Execute pre-start hooks
app.preStartHooks()

33
markdown.go
View File

@ -15,6 +15,7 @@ import (
"github.com/yuin/goldmark/renderer"
"github.com/yuin/goldmark/renderer/html"
"github.com/yuin/goldmark/util"
"go.goblog.app/app/pkgs/bufferpool"
)
func (a *goBlog) initMarkdown() {
@ -88,14 +89,14 @@ func (a *goBlog) renderText(s string) string {
if s == "" {
return ""
}
pipeReader, pipeWriter := io.Pipe()
go func() {
writeErr := a.renderMarkdownToWriter(pipeWriter, s, false)
_ = pipeWriter.CloseWithError(writeErr)
}()
text, readErr := htmlTextFromReader(pipeReader)
_ = pipeReader.CloseWithError(readErr)
if readErr != nil {
buf := bufferpool.Get()
defer bufferpool.Put(buf)
err := a.renderMarkdownToWriter(buf, s, false)
if err != nil {
return ""
}
text, err := htmlTextFromReader(buf)
if err != nil {
return ""
}
return text
@ -105,14 +106,14 @@ func (a *goBlog) renderMdTitle(s string) string {
if s == "" {
return ""
}
pipeReader, pipeWriter := io.Pipe()
go func() {
writeErr := a.titleMd.Convert([]byte(s), pipeWriter)
_ = pipeWriter.CloseWithError(writeErr)
}()
text, readErr := htmlTextFromReader(pipeReader)
_ = pipeReader.CloseWithError(readErr)
if readErr != nil {
buf := bufferpool.Get()
defer bufferpool.Put(buf)
err := a.titleMd.Convert([]byte(s), buf)
if err != nil {
return ""
}
text, err := htmlTextFromReader(buf)
if err != nil {
return ""
}
return text

14
markdown_test.go
View File

@ -108,7 +108,7 @@ func Benchmark_markdown(b *testing.B) {
app.initMarkdown()
b.Run("Benchmark Markdown Rendering", func(b *testing.B) {
b.Run("Markdown Rendering", func(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := app.renderMarkdown(mdExp, true)
if err != nil {
@ -116,4 +116,16 @@ func Benchmark_markdown(b *testing.B) {
}
}
})
b.Run("Title Rendering", func(b *testing.B) {
for i := 0; i < b.N; i++ {
app.renderMdTitle("**Test**")
}
})
b.Run("Text Rendering", func(b *testing.B) {
for i := 0; i < b.N; i++ {
app.renderText("**Test**")
}
})
}

5
notifications.go
View File

@ -7,11 +7,11 @@ import (
"net/http"
"reflect"
"strconv"
"strings"
"time"
"github.com/go-chi/chi/v5"
"github.com/vcraescu/go-paginator"
"go.goblog.app/app/pkgs/bufferpool"
)
const notificationsPath = "/notifications"
@ -57,7 +57,8 @@ type notificationsRequestConfig struct {
}
func buildNotificationsQuery(config *notificationsRequestConfig) (query string, args []interface{}) {
var queryBuilder strings.Builder
queryBuilder := bufferpool.Get()
defer bufferpool.Put(queryBuilder)
queryBuilder.WriteString("select id, time, text from notifications order by id desc")
if config.limit != 0 || config.offset != 0 {
queryBuilder.WriteString(" limit @limit offset @offset")

18
postsDb.go
View File

@ -1,7 +1,6 @@
package main
import (
"bytes"
"database/sql"
"errors"
"fmt"
@ -99,8 +98,9 @@ func (a *goBlog) checkPost(p *post) (err error) {
if err != nil {
return errors.New("failed to parse location template")
}
var pathBuffer bytes.Buffer
err = pathTmpl.Execute(&pathBuffer, map[string]interface{}{
pathBuffer := bufferpool.Get()
defer bufferpool.Put(pathBuffer)
err = pathTmpl.Execute(pathBuffer, map[string]interface{}{
"BlogPath": a.getRelativePath(p.Blog, ""),
"Year": published.Year(),
"Month": int(published.Month()),
@ -171,7 +171,8 @@ func (db *database) savePost(p *post, o *postCreationOptions) error {
db.pcm.Lock()
defer db.pcm.Unlock()
// Build SQL
var sqlBuilder strings.Builder
sqlBuilder := bufferpool.Get()
defer bufferpool.Put(sqlBuilder)
var sqlArgs = []interface{}{dbNoCache}
// Start transaction
sqlBuilder.WriteString("begin;")
@ -294,7 +295,8 @@ func (db *database) replacePostParam(path, param string, values []string) error
db.pcm.Lock()
defer db.pcm.Unlock()
// Build SQL
var sqlBuilder strings.Builder
sqlBuilder := bufferpool.Get()
defer bufferpool.Put(sqlBuilder)
var sqlArgs = []interface{}{dbNoCache}
// Start transaction
sqlBuilder.WriteString("begin;")
@ -343,7 +345,8 @@ type postsRequestConfig struct {
}
func buildPostsQuery(c *postsRequestConfig, selection string) (query string, args []interface{}) {
var queryBuilder strings.Builder
queryBuilder := bufferpool.Get()
defer bufferpool.Put(queryBuilder)
// Selection
queryBuilder.WriteString("select ")
queryBuilder.WriteString(selection)
@ -459,7 +462,8 @@ func (d *database) loadPostParameters(posts []*post, parameters ...string) (err
}
// Build query
sqlArgs := make([]interface{}, 0)
var queryBuilder strings.Builder
queryBuilder := bufferpool.Get()
defer bufferpool.Put(queryBuilder)
queryBuilder.WriteString("select path, parameter, value from post_parameters where")
// Paths
queryBuilder.WriteString(" path in (")

20
templateAssets.go
View File

@ -12,6 +12,7 @@ import (
"strings"
chromahtml "github.com/alecthomas/chroma/formatters/html"
"go.goblog.app/app/pkgs/bufferpool"
"go.goblog.app/app/pkgs/contenttype"
)
@ -115,12 +116,15 @@ func (a *goBlog) initChromaCSS() error {
return err
}
// Generate and minify CSS
pipeReader, pipeWriter := io.Pipe()
go func() {
writeErr := chromahtml.New(chromahtml.ClassPrefix("c-")).WriteCSS(pipeWriter, chromaStyle)
_ = pipeWriter.CloseWithError(writeErr)
}()
readErr := a.compileAsset(chromaPath, pipeReader)
_ = pipeReader.CloseWithError(readErr)
return readErr
buf := bufferpool.Get()
defer bufferpool.Put(buf)
err = chromahtml.New(chromahtml.ClassPrefix("c-")).WriteCSS(buf, chromaStyle)
if err != nil {
return err
}
err = a.compileAsset(chromaPath, buf)
if err != nil {
return err
}
return nil
}

35
tts.go
View File

@ -1,7 +1,6 @@
package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
@ -73,31 +72,29 @@ func (a *goBlog) createPostTTSAudio(p *post) error {
parts = append(parts, strings.Split(htmlText(a.postHtml(p, false)), "\n\n")...)
// Create TTS audio for each part
partsBuffers := make([]io.Reader, len(parts))
var errs []error
var lock sync.Mutex
partWriters := make([]io.Writer, len(parts))
partReaders := make([]io.Reader, len(parts))
for i := range parts {
buf := bufferpool.Get()
defer bufferpool.Put(buf)
partWriters[i] = buf
partReaders[i] = buf
}
errs := make([]error, len(parts))
var wg sync.WaitGroup
for i, part := range parts {
// Increase wait group
wg.Add(1)
go func(i int, part string) {
defer wg.Done()
// Build SSML
ssml := "<speak>" + html.EscapeString(part) + "<break time=\"500ms\"/></speak>"
// Create TTS audio
var audioBuffer bytes.Buffer
err := a.createTTSAudio(lang, ssml, &audioBuffer)
err := a.createTTSAudio(lang, ssml, partWriters[i])
if err != nil {
lock.Lock()
errs = append(errs, err)
lock.Unlock()
errs[i] = err
return
}
// Append buffer to partsBuffers
lock.Lock()
partsBuffers[i] = &audioBuffer
lock.Unlock()
// Decrease wait group
wg.Done()
}(i, part)
}
@ -105,15 +102,17 @@ func (a *goBlog) createPostTTSAudio(p *post) error {
wg.Wait()
// Check if any errors occurred
if len(errs) > 0 {
return errs[0]
for _, err := range errs {
if err != nil {
return err
}
}
// Merge partsBuffers into final buffer
final := bufferpool.Get()
defer bufferpool.Put(final)
hash := sha256.New()
if err := mp3merge.MergeMP3(io.MultiWriter(final, hash), partsBuffers...); err != nil {
if err := mp3merge.MergeMP3(io.MultiWriter(final, hash), partReaders...); err != nil {
return err
}

4
webmention.go
View File

@ -8,6 +8,7 @@ import (
"strings"
"time"
"go.goblog.app/app/pkgs/bufferpool"
"go.goblog.app/app/pkgs/contenttype"
)
@ -226,7 +227,8 @@ type webmentionsRequestConfig struct {
}
func buildWebmentionsQuery(config *webmentionsRequestConfig) (query string, args []interface{}) {
var queryBuilder strings.Builder
queryBuilder := bufferpool.Get()
defer bufferpool.Put(queryBuilder)
queryBuilder.WriteString("select id, source, target, url, created, title, content, author, status from webmentions ")
if config != nil {
queryBuilder.WriteString("where 1")