Browse Source

Use ristretto cache for sql statements and shortpaths

master
Jan-Lukas Else 2 months ago
parent
commit
54de28144c
  1. 35
      database.go
  2. 4
      go.mod
  3. 8
      go.sum
  4. 4
      shortPath.go
  5. 14
      shortPath_test.go

35
database.go

@ -9,6 +9,7 @@ import (
"strings"
"sync"
"github.com/dgraph-io/ristretto"
"github.com/google/uuid"
sqlite "github.com/mattn/go-sqlite3"
"github.com/schollz/sqlite3dump"
@ -17,15 +18,15 @@ import (
type database struct {
// Basic things
db *sql.DB // database
em sync.Mutex // command execution (insert, update, delete ...)
sg singleflight.Group // singleflight group for prepared statements
ps sync.Map // map with prepared statements
db *sql.DB // database
em sync.Mutex // command execution (insert, update, delete ...)
sg singleflight.Group // singleflight group for prepared statements
psc *ristretto.Cache // prepared statement cache
// Other things
pc singleflight.Group // persistant cache
pcm sync.Mutex // post creation
sp singleflight.Group // singleflight group for short path requests
spc sync.Map // shortpath cache
spc *ristretto.Cache // shortpath cache
debug bool
}
@ -120,9 +121,29 @@ func (a *goBlog) openDatabase(file string, logging bool) (*database, error) {
if c := a.cfg.Db; c != nil && c.Debug {
debug = true
}
psc, err := ristretto.NewCache(&ristretto.Config{
NumCounters: 1000,
MaxCost: 100,
BufferItems: 64,
IgnoreInternalCost: true,
})
if err != nil {
return nil, err
}
spc, err := ristretto.NewCache(&ristretto.Config{
NumCounters: 5000,
MaxCost: 500,
BufferItems: 64,
IgnoreInternalCost: true,
})
if err != nil {
return nil, err
}
return &database{
db: db,
debug: debug,
psc: psc,
spc: spc,
}, nil
}
@ -162,7 +183,7 @@ func (db *database) prepare(query string, args ...interface{}) (*sql.Stmt, []int
}
stmt, err, _ := db.sg.Do(query, func() (interface{}, error) {
// Look if statement already exists
st, ok := db.ps.Load(query)
st, ok := db.psc.Get(query)
if ok {
return st, nil
}
@ -172,7 +193,7 @@ func (db *database) prepare(query string, args ...interface{}) (*sql.Stmt, []int
return nil, err
}
// ... and store it
db.ps.Store(query, st)
db.psc.Set(query, st, 1)
return st, nil
})
if err != nil {

4
go.mod

@ -27,12 +27,12 @@ require (
github.com/gorilla/sessions v1.2.1
github.com/gorilla/websocket v1.5.0
github.com/hacdias/indieauth v1.7.1
github.com/jlaffaye/ftp v0.0.0-20220309012535-813c8a838452
github.com/jlaffaye/ftp v0.0.0-20220310202011-d2c44e311e78
// master
github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4
github.com/justinas/alice v1.2.0
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9
github.com/klauspost/compress v1.15.0
github.com/klauspost/compress v1.15.1
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
github.com/lopezator/migrator v0.3.0
github.com/mattn/go-sqlite3 v1.14.12

8
go.sum

@ -253,8 +253,8 @@ github.com/insomniacslk/dhcp v0.0.0-20211026125128-ad197bcd36fd h1:jupbuQFZtwOBg
github.com/insomniacslk/dhcp v0.0.0-20211026125128-ad197bcd36fd/go.mod h1:h+MxyHxRg9NH3terB1nfRIUaQEcI0XOVkdR9LNBlp8E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jlaffaye/ftp v0.0.0-20220309012535-813c8a838452 h1:kqK7woZfi4Y1sIrKu2QxKWa7uDwi1pMeRAzLWG5bWag=
github.com/jlaffaye/ftp v0.0.0-20220309012535-813c8a838452/go.mod h1:oZaomI+9/et52UBjvNU9LCIqmgt816+7ljXCx0EIPzo=
github.com/jlaffaye/ftp v0.0.0-20220310202011-d2c44e311e78 h1:urWv38lDLjDRk5fG9P8vvxlfpQXaKtRlZc+QLKk3FRA=
github.com/jlaffaye/ftp v0.0.0-20220310202011-d2c44e311e78/go.mod h1:oZaomI+9/et52UBjvNU9LCIqmgt816+7ljXCx0EIPzo=
github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4 h1:d2oKwfgLl3ef0PyYDkzjsVyYlBZzNpOpXitDraOnVXc=
github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4/go.mod h1:vt0iOV52/wq97Ql/jp7mUkqyrlEiGQuhHic4bVoHy0c=
github.com/joeshaw/gengen v0.0.0-20190604015154-c77d87825f5a/go.mod h1:v2qvRL8Xwk4OlARK6gPlf2JreZXzv0dYp/8+kUJ0y7Q=
@ -288,8 +288,8 @@ github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9 h1:+9REu9CK9D1AQ
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9/go.mod h1:OvY5ZBrAC9kOvM2PZs9Lw0BH+5K7tjrT6T7SFhn27OA=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U=
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=
github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=

4
shortPath.go

@ -13,7 +13,7 @@ func (db *database) shortenPath(p string) (string, error) {
}
spi, err, _ := db.sp.Do(p, func() (interface{}, error) {
// Check if already cached
if spi, ok := db.spc.Load(p); ok {
if spi, ok := db.spc.Get(p); ok {
return spi.(string), nil
}
// Insert in case it isn't shortened yet
@ -41,7 +41,7 @@ func (db *database) shortenPath(p string) (string, error) {
return nil, err
}
// Cache result
db.spc.Store(p, sp)
db.spc.Set(p, sp, 1)
return sp, nil
})
if err != nil {

14
shortPath_test.go

@ -23,7 +23,7 @@ func Test_shortenPath(t *testing.T) {
res1, err := db.shortenPath("/a")
require.NoError(t, err)
db.spc.Delete("/a")
db.spc.Del("/a")
res2, err := db.shortenPath("/a")
require.NoError(t, err)
@ -31,16 +31,22 @@ func Test_shortenPath(t *testing.T) {
res3, err := db.shortenPath("/b")
require.NoError(t, err)
res4, err := db.shortenPath("/a")
require.NoError(t, err)
assert.Equal(t, res1, res2)
assert.Equal(t, "/s/1", res1)
assert.NotEqual(t, res1, res3)
assert.Equal(t, "/s/2", res3)
db.spc.Delete("/a")
assert.Equal(t, res2, res4)
assert.Equal(t, "/s/1", res4)
db.spc.Del("/a")
_, _ = db.exec("delete from shortpath where id = 1")
res4, err := db.shortenPath("/c")
res5, err := db.shortenPath("/c")
require.NoError(t, err)
assert.Equal(t, "/s/1", res4)
assert.Equal(t, "/s/1", res5)
}

Loading…
Cancel
Save