Add support for IndexNow

app.go

@@ -52,6 +52,9 @@ type goBlog struct {
 	httpClient *http.Client
 	// HTTP Routers
 	d http.Handler
+	// IndexNow
+	inKey  string
+	inLoad singleflight.Group
 	// IndieAuth
 	ias *indieauth.Server
 	// Logs

config.go

@@ -24,6 +24,7 @@ type config struct {
 	Webmention    *configWebmention      `mapstructure:"webmention"`
 	Notifications *configNotifications   `mapstructure:"notifications"`
 	PrivateMode   *configPrivateMode     `mapstructure:"privateMode"`
+	IndexNow      *configIndexNow        `mapstructure:"indexNow"`
 	EasterEgg     *configEasterEgg       `mapstructure:"easterEgg"`
 	Debug         bool                   `mapstructure:"debug"`
 	MapTiles      *configMapTiles        `mapstructure:"mapTiles"`
@@ -279,6 +280,10 @@ type configPrivateMode struct {
 	Enabled bool `mapstructure:"enabled"`
 }
 
+type configIndexNow struct {
+	Enabled bool `mapstructure:"enabled"`
+}
+
 type configEasterEgg struct {
 	Enabled bool `mapstructure:"enabled"`
 }

example-config.yml

@@ -41,6 +41,10 @@ cache:
 privateMode:
   enabled: true # Enable private mode and only allow access with login
 
+# IndexNow (https://www.indexnow.org/index)
+indexNow:
+  enabled: true # Enable IndexNow integration
+
 # User
 user:
   name: John Doe # Full name

go.mod

@@ -56,7 +56,7 @@ require (
 	github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38
 	github.com/yuin/goldmark-highlighting v0.0.0-20210516132338-9216f9c5aa01
 	golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce
-	golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
+	golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/text v0.3.7
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
@@ -130,7 +130,7 @@ require (
 	go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect
 	golang.org/x/mod v0.5.1 // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
 	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
 	golang.org/x/tools v0.1.8 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect

go.sum

@@ -553,8 +553,8 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
-golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba h1:6u6sik+bn/y7vILcYkK3iwTBWN7WtBvB0+SZswQnbf8=
+golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -631,8 +631,8 @@ golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk=
-golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=

http.go

@@ -205,6 +205,13 @@ func (a *goBlog) buildRouter() (http.Handler, error) {
 	// Sitemap
 	r.With(a.privateModeHandler, cacheLoggedIn, a.cacheMiddleware).Get(sitemapPath, a.serveSitemap)
 
+	// IndexNow
+	if a.indexNowEnabled() {
+		if inkey := a.indexNowKey(); inkey != "" {
+			r.With(cacheLoggedIn, a.cacheMiddleware).Get("/"+inkey+".txt", a.serveIndexNow)
+		}
+	}
+
 	// Robots.txt
 	r.With(cacheLoggedIn, a.cacheMiddleware).Get(robotsTXTPath, a.serveRobotsTXT)
 

httpsCache.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"database/sql"
 	"errors"
 
 	"golang.org/x/crypto/acme/autocert"
@@ -27,7 +26,7 @@ func (c *httpsCache) Get(_ context.Context, key string) ([]byte, error) {
 		return nil, err
 	}
 	d, err := c.db.retrievePersistentCache("https_" + key)
-	if errors.Is(err, sql.ErrNoRows) {
+	if d == nil && err == nil {
 		return nil, autocert.ErrCacheMiss
 	} else if err != nil {
 		return nil, err

indexnow.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"context"
+	"log"
+	"net/http"
+
+	"github.com/carlmjohnson/requests"
+	"github.com/thoas/go-funk"
+)
+
+// Implement support for the IndexNow protocol
+// https://www.indexnow.org/documentation
+
+func (a *goBlog) initIndexNow() {
+	if !a.indexNowEnabled() {
+		return
+	}
+	// Add hooks
+	hook := func(p *post) {
+		// Check if post is published
+		if !p.isPublishedSectionPost() {
+			return
+		}
+		// Send IndexNow request
+		a.indexNow(a.fullPostURL(p))
+	}
+	a.pPostHooks = append(a.pPostHooks, hook)
+	a.pUpdateHooks = append(a.pUpdateHooks, hook)
+}
+
+func (a *goBlog) indexNowEnabled() bool {
+	// Check if private mode is enabled
+	if a.isPrivate() {
+		return false
+	}
+	// Check if IndexNow is disabled
+	if inc := a.cfg.IndexNow; inc == nil || !inc.Enabled {
+		return false
+	}
+	return true
+}
+
+func (a *goBlog) serveIndexNow(w http.ResponseWriter, r *http.Request) {
+	w.Write([]byte(a.indexNowKey()))
+}
+
+func (a *goBlog) indexNow(url string) {
+	if !a.indexNowEnabled() {
+		return
+	}
+	key := a.indexNowKey()
+	if key == "" {
+		log.Println("Skipping IndexNow")
+		return
+	}
+	err := requests.URL("https://api.indexnow.org/indexnow").
+		Client(a.httpClient).
+		UserAgent(appUserAgent).
+		Param("url", url).
+		Param("key", key).
+		Fetch(context.Background())
+	if err != nil {
+		log.Println("Sending IndexNow request failed:", err.Error())
+		return
+	} else {
+		log.Println("IndexNow request sent for", url)
+	}
+}
+
+func (a *goBlog) indexNowKey() string {
+	res, _, _ := a.inLoad.Do("", func() (interface{}, error) {
+		// Check if already loaded
+		if a.inKey != "" {
+			return a.inKey, nil
+		}
+		// Try to load key from database
+		keyBytes, err := a.db.retrievePersistentCache("indexnowkey")
+		if err != nil {
+			log.Println("Failed to retrieve cached IndexNow key:", err.Error())
+			return "", err
+		}
+		if keyBytes == nil {
+			// Generate 128 character key with hexadecimal characters
+			keyBytes = []byte(funk.RandomString(128, []rune("0123456789abcdef")))
+			// Store key in database
+			err = a.db.cachePersistently("indexnowkey", keyBytes)
+			if err != nil {
+				log.Println("Failed to cache IndexNow key:", err.Error())
+				return "", err
+			}
+		}
+		a.inKey = string(keyBytes)
+		return a.inKey, nil
+	})
+	return res.(string)
+}

indexnow_test.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_indexNow(t *testing.T) {
+	fc := newFakeHttpClient()
+	fc.setFakeResponse(200, "OK")
+
+	app := &goBlog{
+		cfg:        createDefaultTestConfig(t),
+		httpClient: fc.Client,
+	}
+	app.cfg.IndexNow = &configIndexNow{Enabled: true}
+	_ = app.initConfig()
+	_ = app.initDatabase(false)
+	app.initComponents(false)
+
+	// Create http router
+	app.d, _ = app.buildRouter()
+
+	// Check key
+	require.NotEmpty(t, app.inKey)
+	req, _ := http.NewRequest("GET", "http://localhost:8080/"+app.inKey+".txt", nil)
+	res, err := doHandlerRequest(req, app.d)
+	require.NoError(t, err)
+	require.Equal(t, 200, res.StatusCode)
+	body, _ := io.ReadAll(res.Body)
+	require.Equal(t, app.inKey, string(body))
+
+	// Test publish post
+	_ = app.createPost(&post{
+		Section:   "posts",
+		Path:      "/testpost",
+		Published: "2022-01-01",
+	})
+
+	// Wait for hooks to run
+	time.Sleep(300 * time.Millisecond)
+
+	// Check fake http client
+	require.NotNil(t, fc.req)
+	require.Equal(t, "https://api.indexnow.org/indexnow?key="+app.inKey+"&url=http%3A%2F%2Flocalhost%3A8080%2Ftestpost", fc.req.URL.String())
+}

main.go

@@ -182,6 +182,7 @@ func (app *goBlog) initComponents(logging bool) {
 	app.initIndieAuth()
 	app.startPostsScheduler()
 	app.initPostsDeleter()
+	app.initIndexNow()
 	// Log finish
 	if logging {
 		log.Println("Initialized components")

persistentCache.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"database/sql"
+	"errors"
 )
 
 func (db *database) cachePersistently(key string, data []byte) error {
@@ -11,18 +12,22 @@ func (db *database) cachePersistently(key string, data []byte) error {
 
 func (db *database) retrievePersistentCache(key string) (data []byte, err error) {
 	d, err, _ := db.pc.Do(key, func() (interface{}, error) {
-		if row, err := db.queryRow("select data from persistent_cache where key = @key", sql.Named("key", key)); err == sql.ErrNoRows {
-			return nil, nil
-		} else if err != nil {
+		if row, err := db.queryRow("select data from persistent_cache where key = @key", sql.Named("key", key)); err != nil {
 			return nil, err
 		} else {
 			err = row.Scan(&data)
+			if errors.Is(err, sql.ErrNoRows) {
+				return nil, nil
+			}
 			return data, err
 		}
 	})
 	if err != nil {
 		return nil, err
 	}
+	if d == nil {
+		return nil, nil
+	}
 	return d.([]byte), nil
 }