From 68b2d604c3ace726e7c4cda1aa230f11e5e05b53 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Else Date: Wed, 23 Feb 2022 12:24:11 +0100 Subject: [PATCH] Small changes (less GC, cache stats, linter fixes) --- .golangci.yml | 1 + activityPubSending.go | 2 +- cache.go | 9 +++++++ check.go | 4 --- database.go | 52 +++++++++++++++++++++++---------------- export.go | 1 + garbagecollector.go | 12 ++++++--- httpListener.go | 1 + queue_test.go | 33 ++++++++++++++++++++++++- tailscale.go | 1 + tor.go | 2 +- webmentionVerification.go | 2 +- 12 files changed, 88 insertions(+), 32 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 811e135..f02cc00 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -29,6 +29,7 @@ linters: - bidichk - containedctx - contextcheck + - gosec linters-settings: gosimple: go: "1.17" diff --git a/activityPubSending.go b/activityPubSending.go index 91861f5..c1909cc 100644 --- a/activityPubSending.go +++ b/activityPubSending.go @@ -23,7 +23,7 @@ type apRequest struct { } func (a *goBlog) initAPSendQueue() { - a.listenOnQueue("ap", 15*time.Second, func(qi *queueItem, dequeue func(), reschedule func(time.Duration)) { + a.listenOnQueue("ap", time.Minute, func(qi *queueItem, dequeue func(), reschedule func(time.Duration)) { var r apRequest if err := gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&r); err != nil { log.Println("activitypub queue:", err.Error()) diff --git a/cache.go b/cache.go index 8450192..c4efc3c 100644 --- a/cache.go +++ b/cache.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "fmt" + "log" "net/http" "net/url" "sort" @@ -38,7 +39,15 @@ func (a *goBlog) initCache() (err error) { NumCounters: 40 * 1000, // 4000 items when full with 5 KB items -> x10 = 40.000 MaxCost: 20 * 1000 * 1000, // 20 MB BufferItems: 64, // recommended + Metrics: true, }) + go func() { + ticker := time.NewTicker(15 * time.Minute) + for range ticker.C { + met := a.cache.c.Metrics + log.Println("\nCache:", met.String()) + } + }() return } diff --git a/check.go b/check.go index 15a8788..aa17b77 100644 --- a/check.go +++ b/check.go @@ -2,7 +2,6 @@ package main import ( "context" - "crypto/tls" "fmt" "io" "log" @@ -51,9 +50,6 @@ func (a *goBlog) checkLinks(w io.Writer, posts ...*post) error { client := &http.Client{ Timeout: 30 * time.Second, Transport: gzhttp.Transport(&http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, // Limits DisableKeepAlives: true, MaxConnsPerHost: 1, diff --git a/database.go b/database.go index 8efffb0..11ee447 100644 --- a/database.go +++ b/database.go @@ -210,26 +210,10 @@ func (db *database) execContext(c context.Context, query string, args ...interfa } func (db *database) query(query string, args ...interface{}) (*sql.Rows, error) { - if db == nil || db.db == nil { - return nil, errors.New("database not initialized") - } - // Maybe prepare - st, args, _ := db.prepare(query, args...) - // Prepare context, call hook - ctx := db.dbBefore(context.Background(), query, args...) - defer db.dbAfter(ctx, query, args...) - // Query - if st != nil { - return st.QueryContext(ctx, args...) - } - return db.db.QueryContext(ctx, query, args...) + return db.queryContext(context.Background(), query, args...) } -func (db *database) queryRow(query string, args ...interface{}) (*sql.Row, error) { - return db.queryRowContext(context.Background(), query, args...) -} - -func (db *database) queryRowContext(c context.Context, query string, args ...interface{}) (*sql.Row, error) { +func (db *database) queryContext(c context.Context, query string, args ...interface{}) (rows *sql.Rows, err error) { if db == nil || db.db == nil { return nil, errors.New("database not initialized") } @@ -237,12 +221,38 @@ func (db *database) queryRowContext(c context.Context, query string, args ...int st, args, _ := db.prepare(query, args...) // Prepare context, call hook ctx := db.dbBefore(c, query, args...) - defer db.dbAfter(ctx, query, args...) // Query if st != nil { - return st.QueryRowContext(ctx, args...), nil + rows, err = st.QueryContext(ctx, args...) + } else { + rows, err = db.db.QueryContext(ctx, query, args...) } - return db.db.QueryRowContext(ctx, query, args...), nil + // Call hook + db.dbAfter(ctx, query, args...) + return +} + +func (db *database) queryRow(query string, args ...interface{}) (*sql.Row, error) { + return db.queryRowContext(context.Background(), query, args...) +} + +func (db *database) queryRowContext(c context.Context, query string, args ...interface{}) (row *sql.Row, err error) { + if db == nil || db.db == nil { + return nil, errors.New("database not initialized") + } + // Maybe prepare + st, args, _ := db.prepare(query, args...) + // Prepare context, call hook + ctx := db.dbBefore(c, query, args...) + // Query + if st != nil { + row = st.QueryRowContext(ctx, args...) + } else { + row = db.db.QueryRowContext(ctx, query, args...) + } + // Call hook + db.dbAfter(ctx, query, args...) + return } // Other things diff --git a/export.go b/export.go index be9f85b..09752c5 100644 --- a/export.go +++ b/export.go @@ -17,6 +17,7 @@ func (a *goBlog) exportMarkdownFiles(dir string) error { filename := filepath.Join(dir, p.Path+".md") filedir := filepath.Dir(filename) _ = os.MkdirAll(filedir, 0777) + //nolint:gosec err = os.WriteFile(filename, []byte(p.contentWithParams()), 0666) if err != nil { return err diff --git a/garbagecollector.go b/garbagecollector.go index 76ded2d..049530e 100644 --- a/garbagecollector.go +++ b/garbagecollector.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "log" "runtime" "time" @@ -8,9 +9,9 @@ import ( func initGC() { go func() { - ticker := time.NewTicker(10 * time.Minute) + ticker := time.NewTicker(15 * time.Minute) for range ticker.C { - go doGC() + doGC() } }() } @@ -20,5 +21,10 @@ func doGC() { runtime.ReadMemStats(&old) runtime.GC() runtime.ReadMemStats(&new) - log.Printf("Alloc: %v MiB → %v MiB", old.Alloc/1024/1024, new.Alloc/1024/1024) + log.Println(fmt.Sprintf( + "\nAlloc: %d MiB -> %d MiB\nSys: %d MiB -> %d MiB\nNumGC: %d", + old.Alloc/1024/1024, new.Alloc/1024/1024, + old.Sys/1024/1024, new.Sys/1024/1024, + new.NumGC, + )) } diff --git a/httpListener.go b/httpListener.go index 05eb8e0..3df251e 100644 --- a/httpListener.go +++ b/httpListener.go @@ -41,6 +41,7 @@ func (a *goBlog) getTCPListener(s *http.Server) (net.Listener, error) { } return tls.NewListener(ln, &tls.Config{ GetCertificate: tailscale.GetCertificate, + MinVersion: tls.VersionTLS12, }), nil } else { // Default diff --git a/queue_test.go b/queue_test.go index 9c68a2c..82a19fd 100644 --- a/queue_test.go +++ b/queue_test.go @@ -22,10 +22,12 @@ func Test_queue(t *testing.T) { defer app.db.close() db := app.db + time1 := time.Now() + err := db.enqueue("test", []byte(""), time.Now()) require.Error(t, err) - err = db.enqueue("test", []byte("1"), time.Now()) + err = db.enqueue("test", []byte("1"), time1) require.NoError(t, err) err = db.enqueue("test", []byte("2"), time.Now()) @@ -39,6 +41,7 @@ func Test_queue(t *testing.T) { require.NoError(t, err) require.NotNil(t, qi) require.Equal(t, []byte("1"), qi.content) + require.Equal(t, time1.UTC(), qi.schedule.UTC()) err = db.reschedule(qi, 1*time.Second) require.NoError(t, err) @@ -63,3 +66,31 @@ func Test_queue(t *testing.T) { require.Equal(t, []byte("1"), qi.content) } + +func Benchmark_queue(b *testing.B) { + app := &goBlog{ + cfg: &config{ + Db: &configDb{ + File: filepath.Join(b.TempDir(), "test.db"), + }, + }, + } + _ = app.initDatabase(false) + defer app.db.close() + db := app.db + + err := db.enqueue("test", []byte("1"), time.Now()) + require.NoError(b, err) + + b.Run("Peek with item", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = db.peekQueue(context.Background(), "test") + } + }) + + b.Run("Peek without item", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = db.peekQueue(context.Background(), "abc") + } + }) +} diff --git a/tailscale.go b/tailscale.go index 11ecd60..960dd0d 100644 --- a/tailscale.go +++ b/tailscale.go @@ -49,6 +49,7 @@ func (a *goBlog) getTailscaleListener(addr string) (net.Listener, error) { if addr == ":443" && a.cfg.Server.TailscaleHTTPS { ln = tls.NewListener(ln, &tls.Config{ GetCertificate: tailscale.GetCertificate, + MinVersion: tls.VersionTLS12, }) } return ln, nil diff --git a/tor.go b/tor.go index 4c16ba6..3d8f002 100644 --- a/tor.go +++ b/tor.go @@ -41,7 +41,7 @@ func (a *goBlog) startOnionService(h http.Handler) error { return err } pemEncoded := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: x509Encoded}) - _ = os.WriteFile(torKeyPath, pemEncoded, 0666) + _ = os.WriteFile(torKeyPath, pemEncoded, 0600) } else { d, _ := os.ReadFile(torKeyPath) block, _ := pem.Decode(d) diff --git a/webmentionVerification.go b/webmentionVerification.go index 8e1d8af..912a4de 100644 --- a/webmentionVerification.go +++ b/webmentionVerification.go @@ -21,7 +21,7 @@ import ( ) func (a *goBlog) initWebmentionQueue() { - a.listenOnQueue("wm", 15*time.Second, func(qi *queueItem, dequeue func(), reschedule func(time.Duration)) { + a.listenOnQueue("wm", time.Minute, func(qi *queueItem, dequeue func(), reschedule func(time.Duration)) { var m mention if err := gob.NewDecoder(bytes.NewReader(qi.content)).Decode(&m); err != nil { log.Println("webmention queue:", err.Error())