Improve Cache

cache.go

@@ -1,35 +1,30 @@
 package main
 
 import (
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
-	"sync"
+	"strconv"
 	"time"
 
+	"github.com/araddon/dateparse"
+	lru "github.com/hashicorp/golang-lru"
 	"golang.org/x/sync/singleflight"
 )
 
+const (
+	cacheInternalExpirationHeader = "GoBlog-Expire"
+)
+
 var (
 	cacheGroup singleflight.Group
-	cacheMap   = map[string]*cacheItem{}
+	cacheLru   *lru.Cache
-	cacheMutex = &sync.RWMutex{}
 )
 
-func initCache() {
+func initCache() (err error) {
-	go func() {
+	cacheLru, err = lru.New(200)
-		for {
+	return
-			// GC the entries every 60 seconds
-			time.Sleep(60 * time.Second)
-			cacheMutex.Lock()
-			for key, item := range cacheMap {
-				if item.expired() {
-					delete(cacheMap, key)
-				}
-			}
-			cacheMutex.Unlock()
-		}
-	}()
 }
 
 func cacheMiddleware(next http.Handler) http.Handler {
@@ -38,23 +33,26 @@ func cacheMiddleware(next http.Handler) http.Handler {
 			// check method
 			(r.Method == http.MethodGet || r.Method == http.MethodHead) &&
 			// check bypass query
-			!(r.URL.Query().Get("cache") == "0") {
+			!(r.URL.Query().Get("cache") == "0" || r.URL.Query().Get("cache") == "false") {
 			key := cacheKey(r)
 			// Get cache or render it
 			cacheInterface, _, _ := cacheGroup.Do(key, func() (interface{}, error) {
 				return getCache(key, next, r), nil
 			})
 			cache := cacheInterface.(*cacheItem)
-			// log.Println(string(cache.body))
 			cacheTimeString := time.Unix(cache.creationTime, 0).Format(time.RFC1123)
-			expiresTimeString := time.Unix(cache.creationTime+appConfig.Cache.Expiration, 0).Format(time.RFC1123)
+			expiresTimeString := ""
+			if cache.expiration != 0 {
+				expiresTimeString = time.Unix(cache.creationTime+int64(cache.expiration), 0).Format(time.RFC1123)
+			}
 			// check conditional request
-			ifModifiedSinceHeader := r.Header.Get("If-Modified-Since")
+			if ifModifiedSinceHeader := r.Header.Get("If-Modified-Since"); ifModifiedSinceHeader != "" {
-			if ifModifiedSinceHeader != "" && ifModifiedSinceHeader == cacheTimeString {
+				if t, _ := dateparse.ParseIn(ifModifiedSinceHeader, time.Local); t.Unix() == cache.creationTime {
-				setCacheHeaders(w, cacheTimeString, expiresTimeString)
+					// send 304
-				// send 304
+					setCacheHeaders(w, cacheTimeString, expiresTimeString)
-				w.WriteHeader(http.StatusNotModified)
+					w.WriteHeader(http.StatusNotModified)
-				return
+					return
+				}
 			}
 			// copy cached headers
 			for k, v := range cache.header {
@@ -77,27 +75,37 @@ func cacheKey(r *http.Request) string {
 }
 
 func setCacheHeaders(w http.ResponseWriter, cacheTimeString string, expiresTimeString string) {
-	w.Header().Set("Cache-Control", "public")
+	w.Header().Del(cacheInternalExpirationHeader)
 	w.Header().Set("Last-Modified", cacheTimeString)
-	w.Header().Set("Expires", expiresTimeString)
+	if expiresTimeString != "" {
+		// Set expires time
+		w.Header().Set("Cache-Control", "public")
+		w.Header().Set("Expires", expiresTimeString)
+	} else {
+		w.Header().Set("Cache-Control", fmt.Sprintf("public,max-age=%d", appConfig.Cache.Expiration))
+	}
 }
 
 type cacheItem struct {
 	creationTime int64
+	expiration   int
 	code         int
 	header       http.Header
 	body         []byte
 }
 
 func (c *cacheItem) expired() bool {
-	return c.creationTime < time.Now().Unix()-appConfig.Cache.Expiration
+	if c.expiration != 0 {
+		return c.creationTime < time.Now().Unix()-int64(c.expiration)
+	}
+	return false
 }
 
-func getCache(key string, next http.Handler, r *http.Request) *cacheItem {
+func getCache(key string, next http.Handler, r *http.Request) (item *cacheItem) {
-	cacheMutex.RLock()
+	if lruItem, ok := cacheLru.Get(key); ok {
-	item, ok := cacheMap[key]
+		item = lruItem.(*cacheItem)
-	cacheMutex.RUnlock()
+	}
-	if !ok || item.expired() {
+	if item == nil || item.expired() {
 		// No cache available
 		// Record request
 		recorder := httptest.NewRecorder()
@@ -105,22 +113,24 @@ func getCache(key string, next http.Handler, r *http.Request) *cacheItem {
 		// Cache values from recorder
 		result := recorder.Result()
 		body, _ := ioutil.ReadAll(result.Body)
+		exp, _ := strconv.Atoi(result.Header.Get(cacheInternalExpirationHeader))
 		item = &cacheItem{
 			creationTime: time.Now().Unix(),
+			expiration:   exp,
 			code:         result.StatusCode,
 			header:       result.Header,
 			body:         body,
 		}
 		// Save cache
-		cacheMutex.Lock()
+		cacheLru.Add(key, item)
-		cacheMap[key] = item
-		cacheMutex.Unlock()
 	}
 	return item
 }
 
 func purgeCache() {
-	cacheMutex.Lock()
+	cacheLru.Purge()
-	cacheMap = map[string]*cacheItem{}
+}
-	cacheMutex.Unlock()
+
+func setInternalCacheExpirationHeader(w http.ResponseWriter, expiration int) {
+	w.Header().Set(cacheInternalExpirationHeader, strconv.Itoa(expiration))
 }

config.go

@@ -38,8 +38,8 @@ type configDb struct {
 }
 
 type configCache struct {
-	Enable     bool  `mapstructure:"enable"`
+	Enable     bool `mapstructure:"enable"`
-	Expiration int64 `mapstructure:"expiration"`
+	Expiration int  `mapstructure:"expiration"`
 }
 
 type configBlog struct {
@@ -98,10 +98,11 @@ type search struct {
 }
 
 type customPage struct {
-	Path     string       `mapstructure:"path"`
+	Path            string       `mapstructure:"path"`
-	Template string       `mapstructure:"template"`
+	Template        string       `mapstructure:"template"`
-	Cache    bool         `mapstructure:"cache"`
+	Cache           bool         `mapstructure:"cache"`
-	Data     *interface{} `mapstructure:"data"`
+	CacheExpiration int          `mapstructure:"cacheExpiration"`
+	Data            *interface{} `mapstructure:"data"`
 }
 
 type configUser struct {

customPages.go

@@ -4,6 +4,13 @@ import "net/http"
 
 func serveCustomPage(blog *configBlog, page *customPage) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
+		if appConfig.Cache != nil && appConfig.Cache.Enable && page.Cache {
+			if page.CacheExpiration != 0 {
+				setInternalCacheExpirationHeader(w, page.CacheExpiration)
+			} else {
+				setInternalCacheExpirationHeader(w, int(appConfig.Cache.Expiration))
+			}
+		}
 		render(w, page.Template, &renderData{
 			Blog:      blog,
 			Canonical: appConfig.Server.PublicAddress + page.Path,

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
 	github.com/gorilla/feeds v1.1.1
 	github.com/gorilla/handlers v1.5.1
+	github.com/hashicorp/golang-lru v0.5.4
 	github.com/jeremywohl/flatten v1.0.1
 	github.com/json-iterator/go v1.1.10
 	github.com/klauspost/cpuid v1.3.1 // indirect
@@ -51,9 +52,9 @@ require (
 	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
 	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b // indirect
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
-	golang.org/x/sys v0.0.0-20201116194326-cc9327a14d48 // indirect
+	golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 // indirect
 	golang.org/x/text v0.3.4 // indirect
-	golang.org/x/tools v0.0.0-20201117152513-9036a0f9af11 // indirect
+	golang.org/x/tools v0.0.0-20201120032337-6d151481565c // indirect
 	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect
 	gopkg.in/yaml.v2 v2.3.0 // indirect

go.sum

@@ -134,6 +134,8 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
@@ -427,8 +429,8 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c h1:UIcGWL6/wpCfyGuJnRFJRurA+yj8RrW7Q6x2YMCXt6c=
 golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201116194326-cc9327a14d48 h1:AYCWBZhgIw6XobZ5CibNJr0Rc4ZofGGKvWa1vcx2IGk=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
-golang.org/x/sys v0.0.0-20201116194326-cc9327a14d48/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -465,8 +467,8 @@ golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnf
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20201117152513-9036a0f9af11 h1:gqcmLJzeDSNhSzkyhJ4kxP6CtTimi/5hWFDGp0lFd1w=
+golang.org/x/tools v0.0.0-20201120032337-6d151481565c h1:IXtuZap6vTKIQ3jemmcwf2gY4BT+lwfZHBYwxMGe5/k=
-golang.org/x/tools v0.0.0-20201117152513-9036a0f9af11/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201120032337-6d151481565c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

main.go

@@ -44,6 +44,11 @@ func main() {
 		log.Fatal(err)
 		return
 	}
+	err = initCache()
+	if err != nil {
+		log.Fatal(err)
+		return
+	}
 	err = initRegexRedirects()
 	if err != nil {
 		log.Fatal(err)
@@ -56,7 +61,6 @@ func main() {
 	}
 	initTelegram()
 	initWebmention()
-	initCache()
 	initNodeInfo()
 
 	// Start cron hooks