Use ristretto cache for sql statements and shortpaths

database.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/dgraph-io/ristretto"
 	"github.com/google/uuid"
 	sqlite "github.com/mattn/go-sqlite3"
 	"github.com/schollz/sqlite3dump"
@@ -17,15 +18,15 @@ import (
 
 type database struct {
 	// Basic things
-	db *sql.DB            // database
-	em sync.Mutex         // command execution (insert, update, delete ...)
-	sg singleflight.Group // singleflight group for prepared statements
-	ps sync.Map           // map with prepared statements
+	db  *sql.DB            // database
+	em  sync.Mutex         // command execution (insert, update, delete ...)
+	sg  singleflight.Group // singleflight group for prepared statements
+	psc *ristretto.Cache   // prepared statement cache
 	// Other things
 	pc    singleflight.Group // persistant cache
 	pcm   sync.Mutex         // post creation
 	sp    singleflight.Group // singleflight group for short path requests
-	spc   sync.Map           // shortpath cache
+	spc   *ristretto.Cache   // shortpath cache
 	debug bool
 }
 
@@ -120,9 +121,29 @@ func (a *goBlog) openDatabase(file string, logging bool) (*database, error) {
 	if c := a.cfg.Db; c != nil && c.Debug {
 		debug = true
 	}
+	psc, err := ristretto.NewCache(&ristretto.Config{
+		NumCounters:        1000,
+		MaxCost:            100,
+		BufferItems:        64,
+		IgnoreInternalCost: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+	spc, err := ristretto.NewCache(&ristretto.Config{
+		NumCounters:        5000,
+		MaxCost:            500,
+		BufferItems:        64,
+		IgnoreInternalCost: true,
+	})
+	if err != nil {
+		return nil, err
+	}
 	return &database{
 		db:    db,
 		debug: debug,
+		psc:   psc,
+		spc:   spc,
 	}, nil
 }
 
@@ -162,7 +183,7 @@ func (db *database) prepare(query string, args ...interface{}) (*sql.Stmt, []int
 	}
 	stmt, err, _ := db.sg.Do(query, func() (interface{}, error) {
 		// Look if statement already exists
-		st, ok := db.ps.Load(query)
+		st, ok := db.psc.Get(query)
 		if ok {
 			return st, nil
 		}
@@ -172,7 +193,7 @@ func (db *database) prepare(query string, args ...interface{}) (*sql.Stmt, []int
 			return nil, err
 		}
 		// ... and store it
-		db.ps.Store(query, st)
+		db.psc.Set(query, st, 1)
 		return st, nil
 	})
 	if err != nil {

go.mod

@@ -27,12 +27,12 @@ require (
 	github.com/gorilla/sessions v1.2.1
 	github.com/gorilla/websocket v1.5.0
 	github.com/hacdias/indieauth v1.7.1
-	github.com/jlaffaye/ftp v0.0.0-20220309012535-813c8a838452
+	github.com/jlaffaye/ftp v0.0.0-20220310202011-d2c44e311e78
 	// master
 	github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4
 	github.com/justinas/alice v1.2.0
 	github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9
-	github.com/klauspost/compress v1.15.0
+	github.com/klauspost/compress v1.15.1
 	github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
 	github.com/lopezator/migrator v0.3.0
 	github.com/mattn/go-sqlite3 v1.14.12

go.sum

@@ -253,8 +253,8 @@ github.com/insomniacslk/dhcp v0.0.0-20211026125128-ad197bcd36fd h1:jupbuQFZtwOBg
 github.com/insomniacslk/dhcp v0.0.0-20211026125128-ad197bcd36fd/go.mod h1:h+MxyHxRg9NH3terB1nfRIUaQEcI0XOVkdR9LNBlp8E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/jlaffaye/ftp v0.0.0-20220309012535-813c8a838452 h1:kqK7woZfi4Y1sIrKu2QxKWa7uDwi1pMeRAzLWG5bWag=
-github.com/jlaffaye/ftp v0.0.0-20220309012535-813c8a838452/go.mod h1:oZaomI+9/et52UBjvNU9LCIqmgt816+7ljXCx0EIPzo=
+github.com/jlaffaye/ftp v0.0.0-20220310202011-d2c44e311e78 h1:urWv38lDLjDRk5fG9P8vvxlfpQXaKtRlZc+QLKk3FRA=
+github.com/jlaffaye/ftp v0.0.0-20220310202011-d2c44e311e78/go.mod h1:oZaomI+9/et52UBjvNU9LCIqmgt816+7ljXCx0EIPzo=
 github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4 h1:d2oKwfgLl3ef0PyYDkzjsVyYlBZzNpOpXitDraOnVXc=
 github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4/go.mod h1:vt0iOV52/wq97Ql/jp7mUkqyrlEiGQuhHic4bVoHy0c=
 github.com/joeshaw/gengen v0.0.0-20190604015154-c77d87825f5a/go.mod h1:v2qvRL8Xwk4OlARK6gPlf2JreZXzv0dYp/8+kUJ0y7Q=
@@ -288,8 +288,8 @@ github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9 h1:+9REu9CK9D1AQ
 github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9/go.mod h1:OvY5ZBrAC9kOvM2PZs9Lw0BH+5K7tjrT6T7SFhn27OA=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U=
-github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=
+github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=

shortPath.go

@@ -13,7 +13,7 @@ func (db *database) shortenPath(p string) (string, error) {
 	}
 	spi, err, _ := db.sp.Do(p, func() (interface{}, error) {
 		// Check if already cached
-		if spi, ok := db.spc.Load(p); ok {
+		if spi, ok := db.spc.Get(p); ok {
 			return spi.(string), nil
 		}
 		// Insert in case it isn't shortened yet
@@ -41,7 +41,7 @@ func (db *database) shortenPath(p string) (string, error) {
 			return nil, err
 		}
 		// Cache result
-		db.spc.Store(p, sp)
+		db.spc.Set(p, sp, 1)
 		return sp, nil
 	})
 	if err != nil {

shortPath_test.go

@@ -23,7 +23,7 @@ func Test_shortenPath(t *testing.T) {
 	res1, err := db.shortenPath("/a")
 	require.NoError(t, err)
 
-	db.spc.Delete("/a")
+	db.spc.Del("/a")
 
 	res2, err := db.shortenPath("/a")
 	require.NoError(t, err)
@@ -31,16 +31,22 @@ func Test_shortenPath(t *testing.T) {
 	res3, err := db.shortenPath("/b")
 	require.NoError(t, err)
 
+	res4, err := db.shortenPath("/a")
+	require.NoError(t, err)
+
 	assert.Equal(t, res1, res2)
 	assert.Equal(t, "/s/1", res1)
 
 	assert.NotEqual(t, res1, res3)
 	assert.Equal(t, "/s/2", res3)
 
-	db.spc.Delete("/a")
+	assert.Equal(t, res2, res4)
+	assert.Equal(t, "/s/1", res4)
+
+	db.spc.Del("/a")
 	_, _ = db.exec("delete from shortpath where id = 1")
 
-	res4, err := db.shortenPath("/c")
+	res5, err := db.shortenPath("/c")
 	require.NoError(t, err)
-	assert.Equal(t, "/s/1", res4)
+	assert.Equal(t, "/s/1", res5)
 }