Use more strings.Builder instead of += for building queries etc. (more efficient)

cache.go

@@ -8,15 +8,12 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
-	"net/url"
 	"strconv"
 	"strings"
 	"time"
-	"unsafe"
 
 	"github.com/araddon/dateparse"
 	"github.com/dgraph-io/ristretto"
-	servertiming "github.com/mitchellh/go-server-timing"
 	"golang.org/x/sync/singleflight"
 )
 
@@ -36,23 +33,9 @@ func (a *goBlog) initCache() (err error) {
 		return nil
 	}
 	a.cache.c, err = ristretto.NewCache(&ristretto.Config{
-		NumCounters: 5000,
+		NumCounters: 40 * 1000,        // 4000 items when full with 5 KB items -> x10 = 40.000
-		MaxCost:     20000000, // 20 MB
+		MaxCost:     20 * 1000 * 1000, // 20 MB
-		BufferItems: 16,
+		BufferItems: 64,               // recommended
-		Cost: func(value interface{}) (cost int64) {
-			if cacheItem, ok := value.(*cacheItem); ok {
-				cost = int64(binary.Size(cacheItem.body)) // Byte size of body
-				for h, hv := range cacheItem.header {
-					cost += int64(binary.Size([]byte(h))) // Byte size of header name
-					for _, hvi := range hv {
-						cost += int64(binary.Size([]byte(hvi))) // byte size of each header value item
-					}
-				}
-			} else {
-				cost = int64(unsafe.Sizeof(cacheItem))
-			}
-			return cost
-		},
 	})
 	return
 }
@@ -110,24 +93,21 @@ func (c *cache) cacheMiddleware(next http.Handler) http.Handler {
 }
 
 func cacheKey(r *http.Request) string {
-	key := cacheURLString(r.URL)
+	var buf strings.Builder
 	// Special cases
 	if asRequest, ok := r.Context().Value(asRequestKey).(bool); ok && asRequest {
-		key = "as-" + key
+		buf.WriteString("as-")
 	}
 	if torUsed, ok := r.Context().Value(torUsedKey).(bool); ok && torUsed {
-		key = "tor-" + key
+		buf.WriteString("tor-")
 	}
-	return key
+	// Add cache URL
-}
+	_, _ = buf.WriteString(r.URL.EscapedPath())
-
+	if q := r.URL.Query(); len(q) > 0 {
-func cacheURLString(u *url.URL) string {
-	var buf strings.Builder
-	_, _ = buf.WriteString(u.EscapedPath())
-	if q := u.Query(); len(q) > 0 {
 		_ = buf.WriteByte('?')
 		_, _ = buf.WriteString(q.Encode())
 	}
+	// Return string
 	return buf.String()
 }
 
@@ -152,13 +132,21 @@ type cacheItem struct {
 	body         []byte
 }
 
+// Calculate byte size of cache item using size of body and header
+func (ci *cacheItem) cost() int64 {
+	var headerBuf strings.Builder
+	_ = ci.header.Write(&headerBuf)
+	headerSize := int64(binary.Size(headerBuf.String()))
+	bodySize := int64(binary.Size(ci.body))
+	return headerSize + bodySize
+}
+
 func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *cacheItem) {
 	if rItem, ok := c.c.Get(key); ok {
 		item = rItem.(*cacheItem)
 	}
 	if item == nil {
 		// No cache available
-		servertiming.FromContext(r.Context()).NewMetric("cm")
 		// Remove problematic headers
 		r.Header.Del("If-Modified-Since")
 		r.Header.Del("If-Unmodified-Since")
@@ -202,16 +190,13 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *
 			body:         body,
 		}
 		// Save cache
-		if cch := item.header.Get("Cache-Control"); !strings.Contains(cch, "no-store") && !strings.Contains(cch, "private") && !strings.Contains(cch, "no-cache") {
+		if cch := item.header.Get("Cache-Control"); !containsStrings(cch, "no-store", "private", "no-cache") {
 			if exp == 0 {
-				c.c.Set(key, item, 0)
+				c.c.Set(key, item, item.cost())
 			} else {
-				ttl := time.Duration(exp) * time.Second
+				c.c.SetWithTTL(key, item, item.cost(), time.Duration(exp)*time.Second)
-				c.c.SetWithTTL(key, item, 0, ttl)
 			}
 		}
-	} else {
-		servertiming.FromContext(r.Context()).NewMetric("c")
 	}
 	return item
 }

comments.go

@@ -105,13 +105,13 @@ type commentsRequestConfig struct {
 }
 
 func buildCommentsQuery(config *commentsRequestConfig) (query string, args []interface{}) {
-	args = []interface{}{}
+	var queryBuilder strings.Builder
-	query = "select id, target, name, website, comment from comments order by id desc"
+	queryBuilder.WriteString("select id, target, name, website, comment from comments order by id desc")
 	if config.limit != 0 || config.offset != 0 {
-		query += " limit @limit offset @offset"
+		queryBuilder.WriteString(" limit @limit offset @offset")
 		args = append(args, sql.Named("limit", config.limit), sql.Named("offset", config.offset))
 	}
-	return
+	return queryBuilder.String(), args
 }
 
 func (db *database) getComments(config *commentsRequestConfig) ([]*comment, error) {

http.go

@@ -546,15 +546,18 @@ const blogContextKey contextKey = "blog"
 const pathContextKey contextKey = "httpPath"
 
 func (a *goBlog) refreshCSPDomains() {
-	a.cspDomains = ""
+	var cspBuilder strings.Builder
 	if mp := a.cfg.Micropub.MediaStorage; mp != nil && mp.MediaURL != "" {
 		if u, err := url.Parse(mp.MediaURL); err == nil {
-			a.cspDomains += " " + u.Hostname()
+			cspBuilder.WriteByte(' ')
+			cspBuilder.WriteString(u.Hostname())
 		}
 	}
 	if len(a.cfg.Server.CSPDomains) > 0 {
-		a.cspDomains += " " + strings.Join(a.cfg.Server.CSPDomains, " ")
+		cspBuilder.WriteByte(' ')
+		cspBuilder.WriteString(strings.Join(a.cfg.Server.CSPDomains, " "))
 	}
+	a.cspDomains = cspBuilder.String()
 }
 
 const cspHeader = "Content-Security-Policy"

notifications.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"reflect"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/go-chi/chi/v5"
@@ -54,13 +55,13 @@ type notificationsRequestConfig struct {
 }
 
 func buildNotificationsQuery(config *notificationsRequestConfig) (query string, args []interface{}) {
-	args = []interface{}{}
+	var queryBuilder strings.Builder
-	query = "select id, time, text from notifications order by id desc"
+	queryBuilder.WriteString("select id, time, text from notifications order by id desc")
 	if config.limit != 0 || config.offset != 0 {
-		query += " limit @limit offset @offset"
+		queryBuilder.WriteString(" limit @limit offset @offset")
 		args = append(args, sql.Named("limit", config.limit), sql.Named("offset", config.offset))
 	}
-	return
+	return queryBuilder.String(), args
 }
 
 func (db *database) getNotifications(config *notificationsRequestConfig) ([]*notification, error) {

posts.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"html/template"
 	"net/http"
 	"reflect"
 	"strconv"
@@ -30,8 +31,8 @@ type post struct {
 	Priority   int
 	// Not persisted
 	Slug        string
-	renderCache sync.Map
+	renderCache map[bool]template.HTML
-	renderMutex sync.Mutex
+	renderMutex sync.RWMutex
 }
 
 type postStatus string

postsDb.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"strconv"
 	"strings"
 	"text/template"
 	"time"
@@ -245,113 +246,125 @@ type postsRequestConfig struct {
 }
 
 func buildPostsQuery(c *postsRequestConfig, selection string) (query string, args []interface{}) {
-	args = []interface{}{}
+	var queryBuilder strings.Builder
-	table := "posts"
+	// Selection
+	queryBuilder.WriteString("select ")
+	queryBuilder.WriteString(selection)
+	queryBuilder.WriteString(" from ")
+	// Table
 	if c.search != "" {
-		table = "posts_fts(@search)"
+		queryBuilder.WriteString("posts_fts(@search)")
 		args = append(args, sql.Named("search", c.search))
+	} else {
+		queryBuilder.WriteString("posts")
 	}
-	var wheres []string
+	// Filter
+	queryBuilder.WriteString(" where 1")
 	if c.path != "" {
-		wheres = append(wheres, "path = @path")
+		queryBuilder.WriteString(" and path = @path")
 		args = append(args, sql.Named("path", c.path))
 	}
 	if c.status != "" && c.status != statusNil {
-		wheres = append(wheres, "status = @status")
+		queryBuilder.WriteString(" and status = @status")
 		args = append(args, sql.Named("status", c.status))
 	}
 	if c.blog != "" {
-		wheres = append(wheres, "blog = @blog")
+		queryBuilder.WriteString(" and blog = @blog")
 		args = append(args, sql.Named("blog", c.blog))
 	}
 	if c.parameter != "" {
 		if c.parameterValue != "" {
-			wheres = append(wheres, "path in (select path from post_parameters where parameter = @param and value = @paramval)")
+			queryBuilder.WriteString(" and path in (select path from post_parameters where parameter = @param and value = @paramval)")
 			args = append(args, sql.Named("param", c.parameter), sql.Named("paramval", c.parameterValue))
 		} else {
-			wheres = append(wheres, "path in (select path from post_parameters where parameter = @param and length(coalesce(value, '')) > 0)")
+			queryBuilder.WriteString(" and path in (select path from post_parameters where parameter = @param and length(coalesce(value, '')) > 0)")
 			args = append(args, sql.Named("param", c.parameter))
 		}
 	}
 	if c.taxonomy != nil && len(c.taxonomyValue) > 0 {
-		wheres = append(wheres, "path in (select path from post_parameters where parameter = @taxname and lower(value) = lower(@taxval))")
+		queryBuilder.WriteString(" and path in (select path from post_parameters where parameter = @taxname and lower(value) = lower(@taxval))")
 		args = append(args, sql.Named("taxname", c.taxonomy.Name), sql.Named("taxval", c.taxonomyValue))
 	}
 	if len(c.sections) > 0 {
-		ws := "section in ("
+		queryBuilder.WriteString(" and section in (")
 		for i, section := range c.sections {
 			if i > 0 {
-				ws += ", "
+				queryBuilder.WriteString(", ")
 			}
-			named := fmt.Sprintf("section%v", i)
+			named := "section" + strconv.Itoa(i)
-			ws += "@" + named
+			queryBuilder.WriteByte('@')
+			queryBuilder.WriteString(named)
 			args = append(args, sql.Named(named, section))
 		}
-		ws += ")"
+		queryBuilder.WriteByte(')')
-		wheres = append(wheres, ws)
 	}
 	if c.publishedYear != 0 {
-		wheres = append(wheres, "substr(tolocal(published), 1, 4) = @publishedyear")
+		queryBuilder.WriteString(" and substr(tolocal(published), 1, 4) = @publishedyear")
 		args = append(args, sql.Named("publishedyear", fmt.Sprintf("%0004d", c.publishedYear)))
 	}
 	if c.publishedMonth != 0 {
-		wheres = append(wheres, "substr(tolocal(published), 6, 2) = @publishedmonth")
+		queryBuilder.WriteString(" and substr(tolocal(published), 6, 2) = @publishedmonth")
 		args = append(args, sql.Named("publishedmonth", fmt.Sprintf("%02d", c.publishedMonth)))
 	}
 	if c.publishedDay != 0 {
-		wheres = append(wheres, "substr(tolocal(published), 9, 2) = @publishedday")
+		queryBuilder.WriteString(" and substr(tolocal(published), 9, 2) = @publishedday")
 		args = append(args, sql.Named("publishedday", fmt.Sprintf("%02d", c.publishedDay)))
 	}
-	if len(wheres) > 0 {
+	// Order
-		table += " where " + strings.Join(wheres, " and ")
+	queryBuilder.WriteString(" order by ")
-	}
-	sorting := " order by published desc"
 	if c.randomOrder {
-		sorting = " order by random()"
+		queryBuilder.WriteString("random()")
 	} else if c.priorityOrder {
-		sorting = " order by priority desc, published desc"
+		queryBuilder.WriteString("priority desc, published desc")
+	} else {
+		queryBuilder.WriteString("published desc")
 	}
-	table += sorting
+	// Limit & Offset
 	if c.limit != 0 || c.offset != 0 {
-		table += " limit @limit offset @offset"
+		queryBuilder.WriteString(" limit @limit offset @offset")
 		args = append(args, sql.Named("limit", c.limit), sql.Named("offset", c.offset))
 	}
-	query = "select " + selection + " from " + table
+	return queryBuilder.String(), args
-	return query, args
 }
 
 func (d *database) loadPostParameters(posts []*post, parameters ...string) (err error) {
+	if len(posts) == 0 {
+		return nil
+	}
+	// Build query
 	var sqlArgs []interface{}
-	// Parameter filter
+	var queryBuilder strings.Builder
-	paramFilter := ""
+	queryBuilder.WriteString("select path, parameter, value from post_parameters where")
+	// Paths
+	queryBuilder.WriteString(" path in (")
+	for i, p := range posts {
+		if i > 0 {
+			queryBuilder.WriteString(", ")
+		}
+		named := "path" + strconv.Itoa(i)
+		queryBuilder.WriteByte('@')
+		queryBuilder.WriteString(named)
+		sqlArgs = append(sqlArgs, sql.Named(named, p.Path))
+	}
+	queryBuilder.WriteByte(')')
+	// Parameters
 	if len(parameters) > 0 {
-		paramFilter = " and parameter in ("
+		queryBuilder.WriteString(" and parameter in (")
 		for i, p := range parameters {
 			if i > 0 {
-				paramFilter += ", "
+				queryBuilder.WriteString(", ")
 			}
-			named := fmt.Sprintf("param%v", i)
+			named := "param" + strconv.Itoa(i)
-			paramFilter += "@" + named
+			queryBuilder.WriteByte('@')
+			queryBuilder.WriteString(named)
 			sqlArgs = append(sqlArgs, sql.Named(named, p))
 		}
-		paramFilter += ")"
+		queryBuilder.WriteByte(')')
-	}
-	// Path filter
-	pathFilter := ""
-	if len(posts) > 0 {
-		pathFilter = " and path in ("
-		for i, p := range posts {
-			if i > 0 {
-				pathFilter += ", "
-			}
-			named := fmt.Sprintf("path%v", i)
-			pathFilter += "@" + named
-			sqlArgs = append(sqlArgs, sql.Named(named, p.Path))
-		}
-		pathFilter += ")"
 	}
+	// Order
+	queryBuilder.WriteString(" order by id")
 	// Query
-	rows, err := d.query("select path, parameter, value from post_parameters where 1 = 1"+paramFilter+pathFilter+" order by id", sqlArgs...)
+	rows, err := d.query(queryBuilder.String(), sqlArgs...)
 	if err != nil {
 		return err
 	}
@@ -519,16 +532,18 @@ group by name;
 
 func (db *database) usesOfMediaFile(names ...string) (counts map[string]int, err error) {
 	sqlArgs := []interface{}{dbNoCache}
-	nameValues := ""
+	var nameValues strings.Builder
 	for i, n := range names {
 		if i > 0 {
-			nameValues += ", "
+			nameValues.WriteString(", ")
 		}
-		named := fmt.Sprintf("name%v", i)
+		named := "name" + strconv.Itoa(i)
-		nameValues += fmt.Sprintf("(@%s)", named)
+		nameValues.WriteString("(@")
+		nameValues.WriteString(named)
+		nameValues.WriteByte(')')
 		sqlArgs = append(sqlArgs, sql.Named(named, n))
 	}
-	rows, err := db.query(fmt.Sprintf(mediaUseSql, nameValues), sqlArgs...)
+	rows, err := db.query(fmt.Sprintf(mediaUseSql, nameValues.String()), sqlArgs...)
 	if err != nil {
 		return nil, err
 	}

postsFuncs.go

@@ -48,11 +48,25 @@ func firstPostParameter(p *post, parameter string) string {
 }
 
 func (a *goBlog) postHtml(p *post, absolute bool) template.HTML {
+	p.renderMutex.RLock()
+	// Check cache
+	if r, ok := p.renderCache[absolute]; ok && r != "" {
+		p.renderMutex.RUnlock()
+		return r
+	}
+	p.renderMutex.RUnlock()
+	// No cache, build it...
 	p.renderMutex.Lock()
 	defer p.renderMutex.Unlock()
-	// Check cache
+	// Build HTML
-	if r, ok := p.renderCache.Load(absolute); ok && r != nil {
+	var htmlBuilder strings.Builder
-		return r.(template.HTML)
+	// Add audio to the top
+	if audio, ok := p.Parameters["audio"]; ok && len(audio) > 0 {
+		for _, a := range audio {
+			htmlBuilder.WriteString(`<audio controls preload=none><source src="`)
+			htmlBuilder.WriteString(a)
+			htmlBuilder.WriteString(`"/></audio>`)
+		}
 	}
 	// Render markdown
 	htmlContent, err := a.renderMarkdown(p.Content, absolute)
@@ -60,26 +74,23 @@ func (a *goBlog) postHtml(p *post, absolute bool) template.HTML {
 		log.Fatal(err)
 		return ""
 	}
-	htmlContentStr := string(htmlContent)
+	htmlBuilder.Write(htmlContent)
-	// Add audio to the top
-	if audio, ok := p.Parameters["audio"]; ok && len(audio) > 0 {
-		audios := ""
-		for _, a := range audio {
-			audios += fmt.Sprintf(`<audio controls preload=none><source src="%s"/></audio>`, a)
-		}
-		htmlContentStr = audios + htmlContentStr
-	}
 	// Add links to the bottom
 	if link, ok := p.Parameters["link"]; ok && len(link) > 0 {
-		links := ""
 		for _, l := range link {
-			links += fmt.Sprintf(`<p><a class=u-bookmark-of href="%s" target=_blank rel=noopener>%s</a></p>`, l, l)
+			htmlBuilder.WriteString(`<p><a class=u-bookmark-of href="`)
+			htmlBuilder.WriteString(l)
+			htmlBuilder.WriteString(`" target=_blank rel=noopener>`)
+			htmlBuilder.WriteString(l)
+			htmlBuilder.WriteString(`</a></p>`)
 		}
-		htmlContentStr += links
 	}
 	// Cache
-	html := template.HTML(htmlContentStr)
+	html := template.HTML(htmlBuilder.String())
-	p.renderCache.Store(absolute, html)
+	if p.renderCache == nil {
+		p.renderCache = map[bool]template.HTML{}
+	}
+	p.renderCache[absolute] = html
 	return html
 }
 

utils.go

@@ -247,3 +247,12 @@ func defaultIfEmpty(s, d string) string {
 	}
 	return d
 }
+
+func containsStrings(s string, subStrings ...string) bool {
+	for _, ss := range subStrings {
+		if strings.Contains(s, ss) {
+			return true
+		}
+	}
+	return false
+}

webmention.go

@@ -135,37 +135,38 @@ type webmentionsRequestConfig struct {
 }
 
 func buildWebmentionsQuery(config *webmentionsRequestConfig) (query string, args []interface{}) {
-	args = []interface{}{}
+	var queryBuilder strings.Builder
-	filter := ""
+	queryBuilder.WriteString("select id, source, target, created, title, content, author, status from webmentions ")
 	if config != nil {
-		filter = "where 1 = 1"
+		queryBuilder.WriteString("where 1")
 		if config.target != "" {
-			filter += " and lower(target) = lower(@target)"
+			queryBuilder.WriteString(" and lower(target) = lower(@target)")
 			args = append(args, sql.Named("target", config.target))
 		}
 		if config.status != "" {
-			filter += " and status = @status"
+			queryBuilder.WriteString(" and status = @status")
 			args = append(args, sql.Named("status", config.status))
 		}
 		if config.sourcelike != "" {
-			filter += " and lower(source) like @sourcelike"
+			queryBuilder.WriteString(" and lower(source) like @sourcelike")
 			args = append(args, sql.Named("sourcelike", "%"+strings.ToLower(config.sourcelike)+"%"))
 		}
 		if config.id != 0 {
-			filter += " and id = @id"
+			queryBuilder.WriteString(" and id = @id")
 			args = append(args, sql.Named("id", config.id))
 		}
 	}
-	order := "desc"
+	queryBuilder.WriteString(" order by created ")
 	if config.asc {
-		order = "asc"
+		queryBuilder.WriteString("asc")
+	} else {
+		queryBuilder.WriteString("desc")
 	}
-	query = "select id, source, target, created, title, content, author, status from webmentions " + filter + " order by created " + order
 	if config.limit != 0 || config.offset != 0 {
-		query += " limit @limit offset @offset"
+		queryBuilder.WriteString(" limit @limit offset @offset")
 		args = append(args, sql.Named("limit", config.limit), sql.Named("offset", config.offset))
 	}
-	return query, args
+	return queryBuilder.String(), args
 }
 
 func (db *database) getWebmentions(config *webmentionsRequestConfig) ([]*mention, error) {