From 6429d64b0a229158a7aa98dd98e868daf3467c78 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Else Date: Fri, 23 Jul 2021 17:26:14 +0200 Subject: [PATCH] Use more strings.Builder instead of += for building queries etc. (more efficient) --- cache.go | 59 ++++++++-------------- comments.go | 8 +-- http.go | 9 ++-- notifications.go | 9 ++-- posts.go | 5 +- postsDb.go | 127 ++++++++++++++++++++++++++--------------------- postsFuncs.go | 45 ++++++++++------- utils.go | 9 ++++ webmention.go | 25 +++++----- 9 files changed, 161 insertions(+), 135 deletions(-) diff --git a/cache.go b/cache.go index 276633c..d59f3f8 100644 --- a/cache.go +++ b/cache.go @@ -8,15 +8,12 @@ import ( "io" "net/http" "net/http/httptest" - "net/url" "strconv" "strings" "time" - "unsafe" "github.com/araddon/dateparse" "github.com/dgraph-io/ristretto" - servertiming "github.com/mitchellh/go-server-timing" "golang.org/x/sync/singleflight" ) @@ -36,23 +33,9 @@ func (a *goBlog) initCache() (err error) { return nil } a.cache.c, err = ristretto.NewCache(&ristretto.Config{ - NumCounters: 5000, - MaxCost: 20000000, // 20 MB - BufferItems: 16, - Cost: func(value interface{}) (cost int64) { - if cacheItem, ok := value.(*cacheItem); ok { - cost = int64(binary.Size(cacheItem.body)) // Byte size of body - for h, hv := range cacheItem.header { - cost += int64(binary.Size([]byte(h))) // Byte size of header name - for _, hvi := range hv { - cost += int64(binary.Size([]byte(hvi))) // byte size of each header value item - } - } - } else { - cost = int64(unsafe.Sizeof(cacheItem)) - } - return cost - }, + NumCounters: 40 * 1000, // 4000 items when full with 5 KB items -> x10 = 40.000 + MaxCost: 20 * 1000 * 1000, // 20 MB + BufferItems: 64, // recommended }) return } @@ -110,24 +93,21 @@ func (c *cache) cacheMiddleware(next http.Handler) http.Handler { } func cacheKey(r *http.Request) string { - key := cacheURLString(r.URL) + var buf strings.Builder // Special cases if asRequest, ok := r.Context().Value(asRequestKey).(bool); ok && asRequest { - key = "as-" + key + buf.WriteString("as-") } if torUsed, ok := r.Context().Value(torUsedKey).(bool); ok && torUsed { - key = "tor-" + key + buf.WriteString("tor-") } - return key -} - -func cacheURLString(u *url.URL) string { - var buf strings.Builder - _, _ = buf.WriteString(u.EscapedPath()) - if q := u.Query(); len(q) > 0 { + // Add cache URL + _, _ = buf.WriteString(r.URL.EscapedPath()) + if q := r.URL.Query(); len(q) > 0 { _ = buf.WriteByte('?') _, _ = buf.WriteString(q.Encode()) } + // Return string return buf.String() } @@ -152,13 +132,21 @@ type cacheItem struct { body []byte } +// Calculate byte size of cache item using size of body and header +func (ci *cacheItem) cost() int64 { + var headerBuf strings.Builder + _ = ci.header.Write(&headerBuf) + headerSize := int64(binary.Size(headerBuf.String())) + bodySize := int64(binary.Size(ci.body)) + return headerSize + bodySize +} + func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *cacheItem) { if rItem, ok := c.c.Get(key); ok { item = rItem.(*cacheItem) } if item == nil { // No cache available - servertiming.FromContext(r.Context()).NewMetric("cm") // Remove problematic headers r.Header.Del("If-Modified-Since") r.Header.Del("If-Unmodified-Since") @@ -202,16 +190,13 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item * body: body, } // Save cache - if cch := item.header.Get("Cache-Control"); !strings.Contains(cch, "no-store") && !strings.Contains(cch, "private") && !strings.Contains(cch, "no-cache") { + if cch := item.header.Get("Cache-Control"); !containsStrings(cch, "no-store", "private", "no-cache") { if exp == 0 { - c.c.Set(key, item, 0) + c.c.Set(key, item, item.cost()) } else { - ttl := time.Duration(exp) * time.Second - c.c.SetWithTTL(key, item, 0, ttl) + c.c.SetWithTTL(key, item, item.cost(), time.Duration(exp)*time.Second) } } - } else { - servertiming.FromContext(r.Context()).NewMetric("c") } return item } diff --git a/comments.go b/comments.go index 2b63e5b..4a227ad 100644 --- a/comments.go +++ b/comments.go @@ -105,13 +105,13 @@ type commentsRequestConfig struct { } func buildCommentsQuery(config *commentsRequestConfig) (query string, args []interface{}) { - args = []interface{}{} - query = "select id, target, name, website, comment from comments order by id desc" + var queryBuilder strings.Builder + queryBuilder.WriteString("select id, target, name, website, comment from comments order by id desc") if config.limit != 0 || config.offset != 0 { - query += " limit @limit offset @offset" + queryBuilder.WriteString(" limit @limit offset @offset") args = append(args, sql.Named("limit", config.limit), sql.Named("offset", config.offset)) } - return + return queryBuilder.String(), args } func (db *database) getComments(config *commentsRequestConfig) ([]*comment, error) { diff --git a/http.go b/http.go index 44623c2..2e0bb24 100644 --- a/http.go +++ b/http.go @@ -546,15 +546,18 @@ const blogContextKey contextKey = "blog" const pathContextKey contextKey = "httpPath" func (a *goBlog) refreshCSPDomains() { - a.cspDomains = "" + var cspBuilder strings.Builder if mp := a.cfg.Micropub.MediaStorage; mp != nil && mp.MediaURL != "" { if u, err := url.Parse(mp.MediaURL); err == nil { - a.cspDomains += " " + u.Hostname() + cspBuilder.WriteByte(' ') + cspBuilder.WriteString(u.Hostname()) } } if len(a.cfg.Server.CSPDomains) > 0 { - a.cspDomains += " " + strings.Join(a.cfg.Server.CSPDomains, " ") + cspBuilder.WriteByte(' ') + cspBuilder.WriteString(strings.Join(a.cfg.Server.CSPDomains, " ")) } + a.cspDomains = cspBuilder.String() } const cspHeader = "Content-Security-Policy" diff --git a/notifications.go b/notifications.go index 49aa393..2a1ff3c 100644 --- a/notifications.go +++ b/notifications.go @@ -7,6 +7,7 @@ import ( "net/http" "reflect" "strconv" + "strings" "time" "github.com/go-chi/chi/v5" @@ -54,13 +55,13 @@ type notificationsRequestConfig struct { } func buildNotificationsQuery(config *notificationsRequestConfig) (query string, args []interface{}) { - args = []interface{}{} - query = "select id, time, text from notifications order by id desc" + var queryBuilder strings.Builder + queryBuilder.WriteString("select id, time, text from notifications order by id desc") if config.limit != 0 || config.offset != 0 { - query += " limit @limit offset @offset" + queryBuilder.WriteString(" limit @limit offset @offset") args = append(args, sql.Named("limit", config.limit), sql.Named("offset", config.offset)) } - return + return queryBuilder.String(), args } func (db *database) getNotifications(config *notificationsRequestConfig) ([]*notification, error) { diff --git a/posts.go b/posts.go index b632af9..3acc724 100644 --- a/posts.go +++ b/posts.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "html/template" "net/http" "reflect" "strconv" @@ -30,8 +31,8 @@ type post struct { Priority int // Not persisted Slug string - renderCache sync.Map - renderMutex sync.Mutex + renderCache map[bool]template.HTML + renderMutex sync.RWMutex } type postStatus string diff --git a/postsDb.go b/postsDb.go index 106e527..8d465dd 100644 --- a/postsDb.go +++ b/postsDb.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "strconv" "strings" "text/template" "time" @@ -245,113 +246,125 @@ type postsRequestConfig struct { } func buildPostsQuery(c *postsRequestConfig, selection string) (query string, args []interface{}) { - args = []interface{}{} - table := "posts" + var queryBuilder strings.Builder + // Selection + queryBuilder.WriteString("select ") + queryBuilder.WriteString(selection) + queryBuilder.WriteString(" from ") + // Table if c.search != "" { - table = "posts_fts(@search)" + queryBuilder.WriteString("posts_fts(@search)") args = append(args, sql.Named("search", c.search)) + } else { + queryBuilder.WriteString("posts") } - var wheres []string + // Filter + queryBuilder.WriteString(" where 1") if c.path != "" { - wheres = append(wheres, "path = @path") + queryBuilder.WriteString(" and path = @path") args = append(args, sql.Named("path", c.path)) } if c.status != "" && c.status != statusNil { - wheres = append(wheres, "status = @status") + queryBuilder.WriteString(" and status = @status") args = append(args, sql.Named("status", c.status)) } if c.blog != "" { - wheres = append(wheres, "blog = @blog") + queryBuilder.WriteString(" and blog = @blog") args = append(args, sql.Named("blog", c.blog)) } if c.parameter != "" { if c.parameterValue != "" { - wheres = append(wheres, "path in (select path from post_parameters where parameter = @param and value = @paramval)") + queryBuilder.WriteString(" and path in (select path from post_parameters where parameter = @param and value = @paramval)") args = append(args, sql.Named("param", c.parameter), sql.Named("paramval", c.parameterValue)) } else { - wheres = append(wheres, "path in (select path from post_parameters where parameter = @param and length(coalesce(value, '')) > 0)") + queryBuilder.WriteString(" and path in (select path from post_parameters where parameter = @param and length(coalesce(value, '')) > 0)") args = append(args, sql.Named("param", c.parameter)) } } if c.taxonomy != nil && len(c.taxonomyValue) > 0 { - wheres = append(wheres, "path in (select path from post_parameters where parameter = @taxname and lower(value) = lower(@taxval))") + queryBuilder.WriteString(" and path in (select path from post_parameters where parameter = @taxname and lower(value) = lower(@taxval))") args = append(args, sql.Named("taxname", c.taxonomy.Name), sql.Named("taxval", c.taxonomyValue)) } if len(c.sections) > 0 { - ws := "section in (" + queryBuilder.WriteString(" and section in (") for i, section := range c.sections { if i > 0 { - ws += ", " + queryBuilder.WriteString(", ") } - named := fmt.Sprintf("section%v", i) - ws += "@" + named + named := "section" + strconv.Itoa(i) + queryBuilder.WriteByte('@') + queryBuilder.WriteString(named) args = append(args, sql.Named(named, section)) } - ws += ")" - wheres = append(wheres, ws) + queryBuilder.WriteByte(')') } if c.publishedYear != 0 { - wheres = append(wheres, "substr(tolocal(published), 1, 4) = @publishedyear") + queryBuilder.WriteString(" and substr(tolocal(published), 1, 4) = @publishedyear") args = append(args, sql.Named("publishedyear", fmt.Sprintf("%0004d", c.publishedYear))) } if c.publishedMonth != 0 { - wheres = append(wheres, "substr(tolocal(published), 6, 2) = @publishedmonth") + queryBuilder.WriteString(" and substr(tolocal(published), 6, 2) = @publishedmonth") args = append(args, sql.Named("publishedmonth", fmt.Sprintf("%02d", c.publishedMonth))) } if c.publishedDay != 0 { - wheres = append(wheres, "substr(tolocal(published), 9, 2) = @publishedday") + queryBuilder.WriteString(" and substr(tolocal(published), 9, 2) = @publishedday") args = append(args, sql.Named("publishedday", fmt.Sprintf("%02d", c.publishedDay))) } - if len(wheres) > 0 { - table += " where " + strings.Join(wheres, " and ") - } - sorting := " order by published desc" + // Order + queryBuilder.WriteString(" order by ") if c.randomOrder { - sorting = " order by random()" + queryBuilder.WriteString("random()") } else if c.priorityOrder { - sorting = " order by priority desc, published desc" + queryBuilder.WriteString("priority desc, published desc") + } else { + queryBuilder.WriteString("published desc") } - table += sorting + // Limit & Offset if c.limit != 0 || c.offset != 0 { - table += " limit @limit offset @offset" + queryBuilder.WriteString(" limit @limit offset @offset") args = append(args, sql.Named("limit", c.limit), sql.Named("offset", c.offset)) } - query = "select " + selection + " from " + table - return query, args + return queryBuilder.String(), args } func (d *database) loadPostParameters(posts []*post, parameters ...string) (err error) { + if len(posts) == 0 { + return nil + } + // Build query var sqlArgs []interface{} - // Parameter filter - paramFilter := "" + var queryBuilder strings.Builder + queryBuilder.WriteString("select path, parameter, value from post_parameters where") + // Paths + queryBuilder.WriteString(" path in (") + for i, p := range posts { + if i > 0 { + queryBuilder.WriteString(", ") + } + named := "path" + strconv.Itoa(i) + queryBuilder.WriteByte('@') + queryBuilder.WriteString(named) + sqlArgs = append(sqlArgs, sql.Named(named, p.Path)) + } + queryBuilder.WriteByte(')') + // Parameters if len(parameters) > 0 { - paramFilter = " and parameter in (" + queryBuilder.WriteString(" and parameter in (") for i, p := range parameters { if i > 0 { - paramFilter += ", " + queryBuilder.WriteString(", ") } - named := fmt.Sprintf("param%v", i) - paramFilter += "@" + named + named := "param" + strconv.Itoa(i) + queryBuilder.WriteByte('@') + queryBuilder.WriteString(named) sqlArgs = append(sqlArgs, sql.Named(named, p)) } - paramFilter += ")" - } - // Path filter - pathFilter := "" - if len(posts) > 0 { - pathFilter = " and path in (" - for i, p := range posts { - if i > 0 { - pathFilter += ", " - } - named := fmt.Sprintf("path%v", i) - pathFilter += "@" + named - sqlArgs = append(sqlArgs, sql.Named(named, p.Path)) - } - pathFilter += ")" + queryBuilder.WriteByte(')') } + // Order + queryBuilder.WriteString(" order by id") // Query - rows, err := d.query("select path, parameter, value from post_parameters where 1 = 1"+paramFilter+pathFilter+" order by id", sqlArgs...) + rows, err := d.query(queryBuilder.String(), sqlArgs...) if err != nil { return err } @@ -519,16 +532,18 @@ group by name; func (db *database) usesOfMediaFile(names ...string) (counts map[string]int, err error) { sqlArgs := []interface{}{dbNoCache} - nameValues := "" + var nameValues strings.Builder for i, n := range names { if i > 0 { - nameValues += ", " + nameValues.WriteString(", ") } - named := fmt.Sprintf("name%v", i) - nameValues += fmt.Sprintf("(@%s)", named) + named := "name" + strconv.Itoa(i) + nameValues.WriteString("(@") + nameValues.WriteString(named) + nameValues.WriteByte(')') sqlArgs = append(sqlArgs, sql.Named(named, n)) } - rows, err := db.query(fmt.Sprintf(mediaUseSql, nameValues), sqlArgs...) + rows, err := db.query(fmt.Sprintf(mediaUseSql, nameValues.String()), sqlArgs...) if err != nil { return nil, err } diff --git a/postsFuncs.go b/postsFuncs.go index eb7cd8d..88f251b 100644 --- a/postsFuncs.go +++ b/postsFuncs.go @@ -48,11 +48,25 @@ func firstPostParameter(p *post, parameter string) string { } func (a *goBlog) postHtml(p *post, absolute bool) template.HTML { + p.renderMutex.RLock() + // Check cache + if r, ok := p.renderCache[absolute]; ok && r != "" { + p.renderMutex.RUnlock() + return r + } + p.renderMutex.RUnlock() + // No cache, build it... p.renderMutex.Lock() defer p.renderMutex.Unlock() - // Check cache - if r, ok := p.renderCache.Load(absolute); ok && r != nil { - return r.(template.HTML) + // Build HTML + var htmlBuilder strings.Builder + // Add audio to the top + if audio, ok := p.Parameters["audio"]; ok && len(audio) > 0 { + for _, a := range audio { + htmlBuilder.WriteString(``) + } } // Render markdown htmlContent, err := a.renderMarkdown(p.Content, absolute) @@ -60,26 +74,23 @@ func (a *goBlog) postHtml(p *post, absolute bool) template.HTML { log.Fatal(err) return "" } - htmlContentStr := string(htmlContent) - // Add audio to the top - if audio, ok := p.Parameters["audio"]; ok && len(audio) > 0 { - audios := "" - for _, a := range audio { - audios += fmt.Sprintf(``, a) - } - htmlContentStr = audios + htmlContentStr - } + htmlBuilder.Write(htmlContent) // Add links to the bottom if link, ok := p.Parameters["link"]; ok && len(link) > 0 { - links := "" for _, l := range link { - links += fmt.Sprintf(`

%s

`, l, l) + htmlBuilder.WriteString(`

`) + htmlBuilder.WriteString(l) + htmlBuilder.WriteString(`

`) } - htmlContentStr += links } // Cache - html := template.HTML(htmlContentStr) - p.renderCache.Store(absolute, html) + html := template.HTML(htmlBuilder.String()) + if p.renderCache == nil { + p.renderCache = map[bool]template.HTML{} + } + p.renderCache[absolute] = html return html } diff --git a/utils.go b/utils.go index 6eeb440..acc4a3f 100644 --- a/utils.go +++ b/utils.go @@ -247,3 +247,12 @@ func defaultIfEmpty(s, d string) string { } return d } + +func containsStrings(s string, subStrings ...string) bool { + for _, ss := range subStrings { + if strings.Contains(s, ss) { + return true + } + } + return false +} diff --git a/webmention.go b/webmention.go index 512447a..543cbf9 100644 --- a/webmention.go +++ b/webmention.go @@ -135,37 +135,38 @@ type webmentionsRequestConfig struct { } func buildWebmentionsQuery(config *webmentionsRequestConfig) (query string, args []interface{}) { - args = []interface{}{} - filter := "" + var queryBuilder strings.Builder + queryBuilder.WriteString("select id, source, target, created, title, content, author, status from webmentions ") if config != nil { - filter = "where 1 = 1" + queryBuilder.WriteString("where 1") if config.target != "" { - filter += " and lower(target) = lower(@target)" + queryBuilder.WriteString(" and lower(target) = lower(@target)") args = append(args, sql.Named("target", config.target)) } if config.status != "" { - filter += " and status = @status" + queryBuilder.WriteString(" and status = @status") args = append(args, sql.Named("status", config.status)) } if config.sourcelike != "" { - filter += " and lower(source) like @sourcelike" + queryBuilder.WriteString(" and lower(source) like @sourcelike") args = append(args, sql.Named("sourcelike", "%"+strings.ToLower(config.sourcelike)+"%")) } if config.id != 0 { - filter += " and id = @id" + queryBuilder.WriteString(" and id = @id") args = append(args, sql.Named("id", config.id)) } } - order := "desc" + queryBuilder.WriteString(" order by created ") if config.asc { - order = "asc" + queryBuilder.WriteString("asc") + } else { + queryBuilder.WriteString("desc") } - query = "select id, source, target, created, title, content, author, status from webmentions " + filter + " order by created " + order if config.limit != 0 || config.offset != 0 { - query += " limit @limit offset @offset" + queryBuilder.WriteString(" limit @limit offset @offset") args = append(args, sql.Named("limit", config.limit), sql.Named("offset", config.offset)) } - return query, args + return queryBuilder.String(), args } func (db *database) getWebmentions(config *webmentionsRequestConfig) ([]*mention, error) {