Use ristretto cache for sql statements and shortpaths

This commit is contained in:
Jan-Lukas Else 2022-03-11 10:33:27 +01:00
parent 5f52993a0c
commit 54de28144c
5 changed files with 46 additions and 19 deletions

View File

@ -9,6 +9,7 @@ import (
"strings" "strings"
"sync" "sync"
"github.com/dgraph-io/ristretto"
"github.com/google/uuid" "github.com/google/uuid"
sqlite "github.com/mattn/go-sqlite3" sqlite "github.com/mattn/go-sqlite3"
"github.com/schollz/sqlite3dump" "github.com/schollz/sqlite3dump"
@ -17,15 +18,15 @@ import (
type database struct { type database struct {
// Basic things // Basic things
db *sql.DB // database db *sql.DB // database
em sync.Mutex // command execution (insert, update, delete ...) em sync.Mutex // command execution (insert, update, delete ...)
sg singleflight.Group // singleflight group for prepared statements sg singleflight.Group // singleflight group for prepared statements
ps sync.Map // map with prepared statements psc *ristretto.Cache // prepared statement cache
// Other things // Other things
pc singleflight.Group // persistant cache pc singleflight.Group // persistant cache
pcm sync.Mutex // post creation pcm sync.Mutex // post creation
sp singleflight.Group // singleflight group for short path requests sp singleflight.Group // singleflight group for short path requests
spc sync.Map // shortpath cache spc *ristretto.Cache // shortpath cache
debug bool debug bool
} }
@ -120,9 +121,29 @@ func (a *goBlog) openDatabase(file string, logging bool) (*database, error) {
if c := a.cfg.Db; c != nil && c.Debug { if c := a.cfg.Db; c != nil && c.Debug {
debug = true debug = true
} }
psc, err := ristretto.NewCache(&ristretto.Config{
NumCounters: 1000,
MaxCost: 100,
BufferItems: 64,
IgnoreInternalCost: true,
})
if err != nil {
return nil, err
}
spc, err := ristretto.NewCache(&ristretto.Config{
NumCounters: 5000,
MaxCost: 500,
BufferItems: 64,
IgnoreInternalCost: true,
})
if err != nil {
return nil, err
}
return &database{ return &database{
db: db, db: db,
debug: debug, debug: debug,
psc: psc,
spc: spc,
}, nil }, nil
} }
@ -162,7 +183,7 @@ func (db *database) prepare(query string, args ...interface{}) (*sql.Stmt, []int
} }
stmt, err, _ := db.sg.Do(query, func() (interface{}, error) { stmt, err, _ := db.sg.Do(query, func() (interface{}, error) {
// Look if statement already exists // Look if statement already exists
st, ok := db.ps.Load(query) st, ok := db.psc.Get(query)
if ok { if ok {
return st, nil return st, nil
} }
@ -172,7 +193,7 @@ func (db *database) prepare(query string, args ...interface{}) (*sql.Stmt, []int
return nil, err return nil, err
} }
// ... and store it // ... and store it
db.ps.Store(query, st) db.psc.Set(query, st, 1)
return st, nil return st, nil
}) })
if err != nil { if err != nil {

4
go.mod
View File

@ -27,12 +27,12 @@ require (
github.com/gorilla/sessions v1.2.1 github.com/gorilla/sessions v1.2.1
github.com/gorilla/websocket v1.5.0 github.com/gorilla/websocket v1.5.0
github.com/hacdias/indieauth v1.7.1 github.com/hacdias/indieauth v1.7.1
github.com/jlaffaye/ftp v0.0.0-20220309012535-813c8a838452 github.com/jlaffaye/ftp v0.0.0-20220310202011-d2c44e311e78
// master // master
github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4 github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4
github.com/justinas/alice v1.2.0 github.com/justinas/alice v1.2.0
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9 github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9
github.com/klauspost/compress v1.15.0 github.com/klauspost/compress v1.15.1
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
github.com/lopezator/migrator v0.3.0 github.com/lopezator/migrator v0.3.0
github.com/mattn/go-sqlite3 v1.14.12 github.com/mattn/go-sqlite3 v1.14.12

8
go.sum
View File

@ -253,8 +253,8 @@ github.com/insomniacslk/dhcp v0.0.0-20211026125128-ad197bcd36fd h1:jupbuQFZtwOBg
github.com/insomniacslk/dhcp v0.0.0-20211026125128-ad197bcd36fd/go.mod h1:h+MxyHxRg9NH3terB1nfRIUaQEcI0XOVkdR9LNBlp8E= github.com/insomniacslk/dhcp v0.0.0-20211026125128-ad197bcd36fd/go.mod h1:h+MxyHxRg9NH3terB1nfRIUaQEcI0XOVkdR9LNBlp8E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jlaffaye/ftp v0.0.0-20220309012535-813c8a838452 h1:kqK7woZfi4Y1sIrKu2QxKWa7uDwi1pMeRAzLWG5bWag= github.com/jlaffaye/ftp v0.0.0-20220310202011-d2c44e311e78 h1:urWv38lDLjDRk5fG9P8vvxlfpQXaKtRlZc+QLKk3FRA=
github.com/jlaffaye/ftp v0.0.0-20220309012535-813c8a838452/go.mod h1:oZaomI+9/et52UBjvNU9LCIqmgt816+7ljXCx0EIPzo= github.com/jlaffaye/ftp v0.0.0-20220310202011-d2c44e311e78/go.mod h1:oZaomI+9/et52UBjvNU9LCIqmgt816+7ljXCx0EIPzo=
github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4 h1:d2oKwfgLl3ef0PyYDkzjsVyYlBZzNpOpXitDraOnVXc= github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4 h1:d2oKwfgLl3ef0PyYDkzjsVyYlBZzNpOpXitDraOnVXc=
github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4/go.mod h1:vt0iOV52/wq97Ql/jp7mUkqyrlEiGQuhHic4bVoHy0c= github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4/go.mod h1:vt0iOV52/wq97Ql/jp7mUkqyrlEiGQuhHic4bVoHy0c=
github.com/joeshaw/gengen v0.0.0-20190604015154-c77d87825f5a/go.mod h1:v2qvRL8Xwk4OlARK6gPlf2JreZXzv0dYp/8+kUJ0y7Q= github.com/joeshaw/gengen v0.0.0-20190604015154-c77d87825f5a/go.mod h1:v2qvRL8Xwk4OlARK6gPlf2JreZXzv0dYp/8+kUJ0y7Q=
@ -288,8 +288,8 @@ github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9 h1:+9REu9CK9D1AQ
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9/go.mod h1:OvY5ZBrAC9kOvM2PZs9Lw0BH+5K7tjrT6T7SFhn27OA= github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9/go.mod h1:OvY5ZBrAC9kOvM2PZs9Lw0BH+5K7tjrT6T7SFhn27OA=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U= github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=

View File

@ -13,7 +13,7 @@ func (db *database) shortenPath(p string) (string, error) {
} }
spi, err, _ := db.sp.Do(p, func() (interface{}, error) { spi, err, _ := db.sp.Do(p, func() (interface{}, error) {
// Check if already cached // Check if already cached
if spi, ok := db.spc.Load(p); ok { if spi, ok := db.spc.Get(p); ok {
return spi.(string), nil return spi.(string), nil
} }
// Insert in case it isn't shortened yet // Insert in case it isn't shortened yet
@ -41,7 +41,7 @@ func (db *database) shortenPath(p string) (string, error) {
return nil, err return nil, err
} }
// Cache result // Cache result
db.spc.Store(p, sp) db.spc.Set(p, sp, 1)
return sp, nil return sp, nil
}) })
if err != nil { if err != nil {

View File

@ -23,7 +23,7 @@ func Test_shortenPath(t *testing.T) {
res1, err := db.shortenPath("/a") res1, err := db.shortenPath("/a")
require.NoError(t, err) require.NoError(t, err)
db.spc.Delete("/a") db.spc.Del("/a")
res2, err := db.shortenPath("/a") res2, err := db.shortenPath("/a")
require.NoError(t, err) require.NoError(t, err)
@ -31,16 +31,22 @@ func Test_shortenPath(t *testing.T) {
res3, err := db.shortenPath("/b") res3, err := db.shortenPath("/b")
require.NoError(t, err) require.NoError(t, err)
res4, err := db.shortenPath("/a")
require.NoError(t, err)
assert.Equal(t, res1, res2) assert.Equal(t, res1, res2)
assert.Equal(t, "/s/1", res1) assert.Equal(t, "/s/1", res1)
assert.NotEqual(t, res1, res3) assert.NotEqual(t, res1, res3)
assert.Equal(t, "/s/2", res3) assert.Equal(t, "/s/2", res3)
db.spc.Delete("/a") assert.Equal(t, res2, res4)
assert.Equal(t, "/s/1", res4)
db.spc.Del("/a")
_, _ = db.exec("delete from shortpath where id = 1") _, _ = db.exec("delete from shortpath where id = 1")
res4, err := db.shortenPath("/c") res5, err := db.shortenPath("/c")
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, "/s/1", res4) assert.Equal(t, "/s/1", res5)
} }