jlelse
/
GoBlog
mirror of https://github.com/jlelse/GoBlog
1
Fork 0
Code Packages Releases Activity

Use more strings.Builder instead of += for building queries etc. (more efficient)

This commit is contained in:
Jan-Lukas Else 2021-07-23 17:26:14 +02:00
parent b7f578cf2f
commit 6429d64b0a
9 changed files with 161 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
cache.go
View File

@ -8,15 +8,12 @@ import (
"io"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"time"
"unsafe"
"github.com/araddon/dateparse"
"github.com/dgraph-io/ristretto"
servertiming "github.com/mitchellh/go-server-timing"
"golang.org/x/sync/singleflight"
)
@ -36,23 +33,9 @@ func (a *goBlog) initCache() (err error) {
return nil
}
a.cache.c, err = ristretto.NewCache(&ristretto.Config{
NumCounters: 5000,
MaxCost: 20000000, // 20 MB
BufferItems: 16,
Cost: func(value interface{}) (cost int64) {
if cacheItem, ok := value.(*cacheItem); ok {
cost = int64(binary.Size(cacheItem.body)) // Byte size of body
for h, hv := range cacheItem.header {
cost += int64(binary.Size([]byte(h))) // Byte size of header name
for _, hvi := range hv {
cost += int64(binary.Size([]byte(hvi))) // byte size of each header value item
}
}
} else {
cost = int64(unsafe.Sizeof(cacheItem))
}
return cost
},
NumCounters: 40 * 1000, // 4000 items when full with 5 KB items -> x10 = 40.000
MaxCost: 20 * 1000 * 1000, // 20 MB
BufferItems: 64, // recommended
})
return
}
@ -110,24 +93,21 @@ func (c *cache) cacheMiddleware(next http.Handler) http.Handler {
}
func cacheKey(r *http.Request) string {
key := cacheURLString(r.URL)
var buf strings.Builder
// Special cases
if asRequest, ok := r.Context().Value(asRequestKey).(bool); ok && asRequest {
key = "as-" + key
buf.WriteString("as-")
}
if torUsed, ok := r.Context().Value(torUsedKey).(bool); ok && torUsed {
key = "tor-" + key
buf.WriteString("tor-")
}
return key
}
func cacheURLString(u *url.URL) string {
var buf strings.Builder
_, _ = buf.WriteString(u.EscapedPath())
if q := u.Query(); len(q) > 0 {
// Add cache URL
_, _ = buf.WriteString(r.URL.EscapedPath())
if q := r.URL.Query(); len(q) > 0 {
_ = buf.WriteByte('?')
_, _ = buf.WriteString(q.Encode())
}
// Return string
return buf.String()
}
@ -152,13 +132,21 @@ type cacheItem struct {
body []byte
}
// Calculate byte size of cache item using size of body and header
func (ci *cacheItem) cost() int64 {
var headerBuf strings.Builder
_ = ci.header.Write(&headerBuf)
headerSize := int64(binary.Size(headerBuf.String()))
bodySize := int64(binary.Size(ci.body))
return headerSize + bodySize
}
func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *cacheItem) {
if rItem, ok := c.c.Get(key); ok {
item = rItem.(*cacheItem)
}
if item == nil {
// No cache available
servertiming.FromContext(r.Context()).NewMetric("cm")
// Remove problematic headers
r.Header.Del("If-Modified-Since")
r.Header.Del("If-Unmodified-Since")
@ -202,16 +190,13 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *
body: body,
}
// Save cache
if cch := item.header.Get("Cache-Control"); !strings.Contains(cch, "no-store") && !strings.Contains(cch, "private") && !strings.Contains(cch, "no-cache") {
if cch := item.header.Get("Cache-Control"); !containsStrings(cch, "no-store", "private", "no-cache") {
if exp == 0 {
c.c.Set(key, item, 0)
c.c.Set(key, item, item.cost())
} else {
ttl := time.Duration(exp) * time.Second
c.c.SetWithTTL(key, item, 0, ttl)
c.c.SetWithTTL(key, item, item.cost(), time.Duration(exp)*time.Second)
}
}
} else {
servertiming.FromContext(r.Context()).NewMetric("c")
}
return item
}

8
comments.go
View File

@ -105,13 +105,13 @@ type commentsRequestConfig struct {
}
func buildCommentsQuery(config *commentsRequestConfig) (query string, args []interface{}) {
args = []interface{}{}
query = "select id, target, name, website, comment from comments order by id desc"
var queryBuilder strings.Builder
queryBuilder.WriteString("select id, target, name, website, comment from comments order by id desc")
if config.limit != 0 || config.offset != 0 {
query += " limit @limit offset @offset"
queryBuilder.WriteString(" limit @limit offset @offset")
args = append(args, sql.Named("limit", config.limit), sql.Named("offset", config.offset))
}
return
return queryBuilder.String(), args
}
func (db *database) getComments(config *commentsRequestConfig) ([]*comment, error) {

9
http.go
View File

@ -546,15 +546,18 @@ const blogContextKey contextKey = "blog"
const pathContextKey contextKey = "httpPath"
func (a *goBlog) refreshCSPDomains() {
a.cspDomains = ""
var cspBuilder strings.Builder
if mp := a.cfg.Micropub.MediaStorage; mp != nil && mp.MediaURL != "" {
if u, err := url.Parse(mp.MediaURL); err == nil {
a.cspDomains += " " + u.Hostname()
cspBuilder.WriteByte(' ')
cspBuilder.WriteString(u.Hostname())
}
}
if len(a.cfg.Server.CSPDomains) > 0 {
a.cspDomains += " " + strings.Join(a.cfg.Server.CSPDomains, " ")
cspBuilder.WriteByte(' ')
cspBuilder.WriteString(strings.Join(a.cfg.Server.CSPDomains, " "))
}
a.cspDomains = cspBuilder.String()
}
const cspHeader = "Content-Security-Policy"

9
notifications.go
View File

@ -7,6 +7,7 @@ import (
"net/http"
"reflect"
"strconv"
"strings"
"time"
"github.com/go-chi/chi/v5"
@ -54,13 +55,13 @@ type notificationsRequestConfig struct {
}
func buildNotificationsQuery(config *notificationsRequestConfig) (query string, args []interface{}) {
args = []interface{}{}
query = "select id, time, text from notifications order by id desc"
var queryBuilder strings.Builder
queryBuilder.WriteString("select id, time, text from notifications order by id desc")
if config.limit != 0 || config.offset != 0 {
query += " limit @limit offset @offset"
queryBuilder.WriteString(" limit @limit offset @offset")
args = append(args, sql.Named("limit", config.limit), sql.Named("offset", config.offset))
}
return
return queryBuilder.String(), args
}
func (db *database) getNotifications(config *notificationsRequestConfig) ([]*notification, error) {

5
posts.go
View File

@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"html/template"
"net/http"
"reflect"
"strconv"
@ -30,8 +31,8 @@ type post struct {
Priority int
// Not persisted
Slug string
renderCache sync.Map
renderMutex sync.Mutex
renderCache map[bool]template.HTML
renderMutex sync.RWMutex
}
type postStatus string

129
postsDb.go
View File

@ -5,6 +5,7 @@ import (
"database/sql"
"errors"
"fmt"
"strconv"
"strings"
"text/template"
"time"
@ -245,113 +246,125 @@ type postsRequestConfig struct {
}
func buildPostsQuery(c *postsRequestConfig, selection string) (query string, args []interface{}) {
args = []interface{}{}
table := "posts"
var queryBuilder strings.Builder
// Selection
queryBuilder.WriteString("select ")
queryBuilder.WriteString(selection)
queryBuilder.WriteString(" from ")
// Table
if c.search != "" {
table = "posts_fts(@search)"
queryBuilder.WriteString("posts_fts(@search)")
args = append(args, sql.Named("search", c.search))
} else {
queryBuilder.WriteString("posts")
}
var wheres []string
// Filter
queryBuilder.WriteString(" where 1")
if c.path != "" {
wheres = append(wheres, "path = @path")
queryBuilder.WriteString(" and path = @path")
args = append(args, sql.Named("path", c.path))
}
if c.status != "" && c.status != statusNil {
wheres = append(wheres, "status = @status")
queryBuilder.WriteString(" and status = @status")
args = append(args, sql.Named("status", c.status))
}
if c.blog != "" {
wheres = append(wheres, "blog = @blog")
queryBuilder.WriteString(" and blog = @blog")
args = append(args, sql.Named("blog", c.blog))
}
if c.parameter != "" {
if c.parameterValue != "" {
wheres = append(wheres, "path in (select path from post_parameters where parameter = @param and value = @paramval)")
queryBuilder.WriteString(" and path in (select path from post_parameters where parameter = @param and value = @paramval)")
args = append(args, sql.Named("param", c.parameter), sql.Named("paramval", c.parameterValue))
} else {
wheres = append(wheres, "path in (select path from post_parameters where parameter = @param and length(coalesce(value, '')) > 0)")
queryBuilder.WriteString(" and path in (select path from post_parameters where parameter = @param and length(coalesce(value, '')) > 0)")
args = append(args, sql.Named("param", c.parameter))
}
}
if c.taxonomy != nil && len(c.taxonomyValue) > 0 {
wheres = append(wheres, "path in (select path from post_parameters where parameter = @taxname and lower(value) = lower(@taxval))")
queryBuilder.WriteString(" and path in (select path from post_parameters where parameter = @taxname and lower(value) = lower(@taxval))")
args = append(args, sql.Named("taxname", c.taxonomy.Name), sql.Named("taxval", c.taxonomyValue))
}
if len(c.sections) > 0 {
ws := "section in ("
queryBuilder.WriteString(" and section in (")
for i, section := range c.sections {
if i > 0 {
ws += ", "
queryBuilder.WriteString(", ")
}
named := fmt.Sprintf("section%v", i)
ws += "@" + named
named := "section" + strconv.Itoa(i)
queryBuilder.WriteByte('@')
queryBuilder.WriteString(named)
args = append(args, sql.Named(named, section))
}
ws += ")"
wheres = append(wheres, ws)
queryBuilder.WriteByte(')')
}
if c.publishedYear != 0 {
wheres = append(wheres, "substr(tolocal(published), 1, 4) = @publishedyear")
queryBuilder.WriteString(" and substr(tolocal(published), 1, 4) = @publishedyear")
args = append(args, sql.Named("publishedyear", fmt.Sprintf("%0004d", c.publishedYear)))
}
if c.publishedMonth != 0 {
wheres = append(wheres, "substr(tolocal(published), 6, 2) = @publishedmonth")
queryBuilder.WriteString(" and substr(tolocal(published), 6, 2) = @publishedmonth")
args = append(args, sql.Named("publishedmonth", fmt.Sprintf("%02d", c.publishedMonth)))
}
if c.publishedDay != 0 {
wheres = append(wheres, "substr(tolocal(published), 9, 2) = @publishedday")
queryBuilder.WriteString(" and substr(tolocal(published), 9, 2) = @publishedday")
args = append(args, sql.Named("publishedday", fmt.Sprintf("%02d", c.publishedDay)))
}
if len(wheres) > 0 {
table += " where " + strings.Join(wheres, " and ")
}
sorting := " order by published desc"
// Order
queryBuilder.WriteString(" order by ")
if c.randomOrder {
sorting = " order by random()"
queryBuilder.WriteString("random()")
} else if c.priorityOrder {
sorting = " order by priority desc, published desc"
queryBuilder.WriteString("priority desc, published desc")
} else {
queryBuilder.WriteString("published desc")
}
table += sorting
// Limit & Offset
if c.limit != 0 || c.offset != 0 {
table += " limit @limit offset @offset"
queryBuilder.WriteString(" limit @limit offset @offset")
args = append(args, sql.Named("limit", c.limit), sql.Named("offset", c.offset))
}
query = "select " + selection + " from " + table
return query, args
return queryBuilder.String(), args
}
func (d *database) loadPostParameters(posts []*post, parameters ...string) (err error) {
if len(posts) == 0 {
return nil
}
// Build query
var sqlArgs []interface{}
// Parameter filter
paramFilter := ""
if len(parameters) > 0 {
paramFilter = " and parameter in ("
for i, p := range parameters {
if i > 0 {
paramFilter += ", "
}
named := fmt.Sprintf("param%v", i)
paramFilter += "@" + named
sqlArgs = append(sqlArgs, sql.Named(named, p))
}
paramFilter += ")"
}
// Path filter
pathFilter := ""
if len(posts) > 0 {
pathFilter = " and path in ("
var queryBuilder strings.Builder
queryBuilder.WriteString("select path, parameter, value from post_parameters where")
// Paths
queryBuilder.WriteString(" path in (")
for i, p := range posts {
if i > 0 {
pathFilter += ", "
queryBuilder.WriteString(", ")
}
named := fmt.Sprintf("path%v", i)
pathFilter += "@" + named
named := "path" + strconv.Itoa(i)
queryBuilder.WriteByte('@')
queryBuilder.WriteString(named)
sqlArgs = append(sqlArgs, sql.Named(named, p.Path))
}
pathFilter += ")"
queryBuilder.WriteByte(')')
// Parameters
if len(parameters) > 0 {
queryBuilder.WriteString(" and parameter in (")
for i, p := range parameters {
if i > 0 {
queryBuilder.WriteString(", ")
}
named := "param" + strconv.Itoa(i)
queryBuilder.WriteByte('@')
queryBuilder.WriteString(named)
sqlArgs = append(sqlArgs, sql.Named(named, p))
}
queryBuilder.WriteByte(')')
}
// Order
queryBuilder.WriteString(" order by id")
// Query
rows, err := d.query("select path, parameter, value from post_parameters where 1 = 1"+paramFilter+pathFilter+" order by id", sqlArgs...)
rows, err := d.query(queryBuilder.String(), sqlArgs...)
if err != nil {
return err
}
@ -519,16 +532,18 @@ group by name;
func (db *database) usesOfMediaFile(names ...string) (counts map[string]int, err error) {
sqlArgs := []interface{}{dbNoCache}
nameValues := ""
var nameValues strings.Builder
for i, n := range names {
if i > 0 {
nameValues += ", "
nameValues.WriteString(", ")
}
named := fmt.Sprintf("name%v", i)
nameValues += fmt.Sprintf("(@%s)", named)
named := "name" + strconv.Itoa(i)
nameValues.WriteString("(@")
nameValues.WriteString(named)
nameValues.WriteByte(')')
sqlArgs = append(sqlArgs, sql.Named(named, n))
}
rows, err := db.query(fmt.Sprintf(mediaUseSql, nameValues), sqlArgs...)
rows, err := db.query(fmt.Sprintf(mediaUseSql, nameValues.String()), sqlArgs...)
if err != nil {
return nil, err
}

45
postsFuncs.go
View File

@ -48,11 +48,25 @@ func firstPostParameter(p *post, parameter string) string {
}
func (a *goBlog) postHtml(p *post, absolute bool) template.HTML {
p.renderMutex.RLock()
// Check cache
if r, ok := p.renderCache[absolute]; ok && r != "" {
p.renderMutex.RUnlock()
return r
}
p.renderMutex.RUnlock()
// No cache, build it...
p.renderMutex.Lock()
defer p.renderMutex.Unlock()
// Check cache
if r, ok := p.renderCache.Load(absolute); ok && r != nil {
return r.(template.HTML)
// Build HTML
var htmlBuilder strings.Builder
// Add audio to the top
if audio, ok := p.Parameters["audio"]; ok && len(audio) > 0 {
for _, a := range audio {
htmlBuilder.WriteString(`<audio controls preload=none><source src="`)
htmlBuilder.WriteString(a)
htmlBuilder.WriteString(`"/></audio>`)
}
}
// Render markdown
htmlContent, err := a.renderMarkdown(p.Content, absolute)
@ -60,26 +74,23 @@ func (a *goBlog) postHtml(p *post, absolute bool) template.HTML {
log.Fatal(err)
return ""
}
htmlContentStr := string(htmlContent)
// Add audio to the top
if audio, ok := p.Parameters["audio"]; ok && len(audio) > 0 {
audios := ""
for _, a := range audio {
audios += fmt.Sprintf(`<audio controls preload=none><source src="%s"/></audio>`, a)
}
htmlContentStr = audios + htmlContentStr
}
htmlBuilder.Write(htmlContent)
// Add links to the bottom
if link, ok := p.Parameters["link"]; ok && len(link) > 0 {
links := ""
for _, l := range link {
links += fmt.Sprintf(`<p><a class=u-bookmark-of href="%s" target=_blank rel=noopener>%s</a></p>`, l, l)
htmlBuilder.WriteString(`<p><a class=u-bookmark-of href="`)
htmlBuilder.WriteString(l)
htmlBuilder.WriteString(`" target=_blank rel=noopener>`)
htmlBuilder.WriteString(l)
htmlBuilder.WriteString(`</a></p>`)
}
htmlContentStr += links
}
// Cache
html := template.HTML(htmlContentStr)
p.renderCache.Store(absolute, html)
html := template.HTML(htmlBuilder.String())
if p.renderCache == nil {
p.renderCache = map[bool]template.HTML{}
}
p.renderCache[absolute] = html
return html
}

9
utils.go
View File

@ -247,3 +247,12 @@ func defaultIfEmpty(s, d string) string {
}
return d
}
func containsStrings(s string, subStrings ...string) bool {
for _, ss := range subStrings {
if strings.Contains(s, ss) {
return true
}
}
return false
}

25
webmention.go
View File

@ -135,37 +135,38 @@ type webmentionsRequestConfig struct {
}
func buildWebmentionsQuery(config *webmentionsRequestConfig) (query string, args []interface{}) {
args = []interface{}{}
filter := ""
var queryBuilder strings.Builder
queryBuilder.WriteString("select id, source, target, created, title, content, author, status from webmentions ")
if config != nil {
filter = "where 1 = 1"
queryBuilder.WriteString("where 1")
if config.target != "" {
filter += " and lower(target) = lower(@target)"
queryBuilder.WriteString(" and lower(target) = lower(@target)")
args = append(args, sql.Named("target", config.target))
}
if config.status != "" {
filter += " and status = @status"
queryBuilder.WriteString(" and status = @status")
args = append(args, sql.Named("status", config.status))
}
if config.sourcelike != "" {
filter += " and lower(source) like @sourcelike"
queryBuilder.WriteString(" and lower(source) like @sourcelike")
args = append(args, sql.Named("sourcelike", "%"+strings.ToLower(config.sourcelike)+"%"))
}
if config.id != 0 {
filter += " and id = @id"
queryBuilder.WriteString(" and id = @id")
args = append(args, sql.Named("id", config.id))
}
}
order := "desc"
queryBuilder.WriteString(" order by created ")
if config.asc {
order = "asc"
queryBuilder.WriteString("asc")
} else {
queryBuilder.WriteString("desc")
}
query = "select id, source, target, created, title, content, author, status from webmentions " + filter + " order by created " + order
if config.limit != 0 || config.offset != 0 {
query += " limit @limit offset @offset"
queryBuilder.WriteString(" limit @limit offset @offset")
args = append(args, sql.Named("limit", config.limit), sql.Named("offset", config.offset))
}
return query, args
return queryBuilder.String(), args
}
func (db *database) getWebmentions(config *webmentionsRequestConfig) ([]*mention, error) {