diff --git a/app.go b/app.go index a415ef8..006fe4c 100644 --- a/app.go +++ b/app.go @@ -52,6 +52,9 @@ type goBlog struct { httpClient *http.Client // HTTP Routers d http.Handler + // IndexNow + inKey string + inLoad singleflight.Group // IndieAuth ias *indieauth.Server // Logs diff --git a/config.go b/config.go index 706fd10..cee26a9 100644 --- a/config.go +++ b/config.go @@ -24,6 +24,7 @@ type config struct { Webmention *configWebmention `mapstructure:"webmention"` Notifications *configNotifications `mapstructure:"notifications"` PrivateMode *configPrivateMode `mapstructure:"privateMode"` + IndexNow *configIndexNow `mapstructure:"indexNow"` EasterEgg *configEasterEgg `mapstructure:"easterEgg"` Debug bool `mapstructure:"debug"` MapTiles *configMapTiles `mapstructure:"mapTiles"` @@ -279,6 +280,10 @@ type configPrivateMode struct { Enabled bool `mapstructure:"enabled"` } +type configIndexNow struct { + Enabled bool `mapstructure:"enabled"` +} + type configEasterEgg struct { Enabled bool `mapstructure:"enabled"` } diff --git a/example-config.yml b/example-config.yml index fa8041f..02af752 100644 --- a/example-config.yml +++ b/example-config.yml @@ -41,6 +41,10 @@ cache: privateMode: enabled: true # Enable private mode and only allow access with login +# IndexNow (https://www.indexnow.org/index) +indexNow: + enabled: true # Enable IndexNow integration + # User user: name: John Doe # Full name diff --git a/go.mod b/go.mod index dc5b18c..c207770 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38 github.com/yuin/goldmark-highlighting v0.0.0-20210516132338-9216f9c5aa01 golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce - golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d + golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/text v0.3.7 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b @@ -130,7 +130,7 @@ require ( go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect golang.org/x/mod v0.5.1 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect golang.org/x/tools v0.1.8 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/go.sum b/go.sum index c17cf29..22117b7 100644 --- a/go.sum +++ b/go.sum @@ -553,8 +553,8 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs= -golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba h1:6u6sik+bn/y7vILcYkK3iwTBWN7WtBvB0+SZswQnbf8= +golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -631,8 +631,8 @@ golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= diff --git a/http.go b/http.go index 12d938a..1598bb1 100644 --- a/http.go +++ b/http.go @@ -205,6 +205,13 @@ func (a *goBlog) buildRouter() (http.Handler, error) { // Sitemap r.With(a.privateModeHandler, cacheLoggedIn, a.cacheMiddleware).Get(sitemapPath, a.serveSitemap) + // IndexNow + if a.indexNowEnabled() { + if inkey := a.indexNowKey(); inkey != "" { + r.With(cacheLoggedIn, a.cacheMiddleware).Get("/"+inkey+".txt", a.serveIndexNow) + } + } + // Robots.txt r.With(cacheLoggedIn, a.cacheMiddleware).Get(robotsTXTPath, a.serveRobotsTXT) diff --git a/httpsCache.go b/httpsCache.go index 05dfc4d..b47a410 100644 --- a/httpsCache.go +++ b/httpsCache.go @@ -2,7 +2,6 @@ package main import ( "context" - "database/sql" "errors" "golang.org/x/crypto/acme/autocert" @@ -27,7 +26,7 @@ func (c *httpsCache) Get(_ context.Context, key string) ([]byte, error) { return nil, err } d, err := c.db.retrievePersistentCache("https_" + key) - if errors.Is(err, sql.ErrNoRows) { + if d == nil && err == nil { return nil, autocert.ErrCacheMiss } else if err != nil { return nil, err diff --git a/indexnow.go b/indexnow.go new file mode 100644 index 0000000..b9be4b7 --- /dev/null +++ b/indexnow.go @@ -0,0 +1,97 @@ +package main + +import ( + "context" + "log" + "net/http" + + "github.com/carlmjohnson/requests" + "github.com/thoas/go-funk" +) + +// Implement support for the IndexNow protocol +// https://www.indexnow.org/documentation + +func (a *goBlog) initIndexNow() { + if !a.indexNowEnabled() { + return + } + // Add hooks + hook := func(p *post) { + // Check if post is published + if !p.isPublishedSectionPost() { + return + } + // Send IndexNow request + a.indexNow(a.fullPostURL(p)) + } + a.pPostHooks = append(a.pPostHooks, hook) + a.pUpdateHooks = append(a.pUpdateHooks, hook) +} + +func (a *goBlog) indexNowEnabled() bool { + // Check if private mode is enabled + if a.isPrivate() { + return false + } + // Check if IndexNow is disabled + if inc := a.cfg.IndexNow; inc == nil || !inc.Enabled { + return false + } + return true +} + +func (a *goBlog) serveIndexNow(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(a.indexNowKey())) +} + +func (a *goBlog) indexNow(url string) { + if !a.indexNowEnabled() { + return + } + key := a.indexNowKey() + if key == "" { + log.Println("Skipping IndexNow") + return + } + err := requests.URL("https://api.indexnow.org/indexnow"). + Client(a.httpClient). + UserAgent(appUserAgent). + Param("url", url). + Param("key", key). + Fetch(context.Background()) + if err != nil { + log.Println("Sending IndexNow request failed:", err.Error()) + return + } else { + log.Println("IndexNow request sent for", url) + } +} + +func (a *goBlog) indexNowKey() string { + res, _, _ := a.inLoad.Do("", func() (interface{}, error) { + // Check if already loaded + if a.inKey != "" { + return a.inKey, nil + } + // Try to load key from database + keyBytes, err := a.db.retrievePersistentCache("indexnowkey") + if err != nil { + log.Println("Failed to retrieve cached IndexNow key:", err.Error()) + return "", err + } + if keyBytes == nil { + // Generate 128 character key with hexadecimal characters + keyBytes = []byte(funk.RandomString(128, []rune("0123456789abcdef"))) + // Store key in database + err = a.db.cachePersistently("indexnowkey", keyBytes) + if err != nil { + log.Println("Failed to cache IndexNow key:", err.Error()) + return "", err + } + } + a.inKey = string(keyBytes) + return a.inKey, nil + }) + return res.(string) +} diff --git a/indexnow_test.go b/indexnow_test.go new file mode 100644 index 0000000..307cad2 --- /dev/null +++ b/indexnow_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "io" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func Test_indexNow(t *testing.T) { + fc := newFakeHttpClient() + fc.setFakeResponse(200, "OK") + + app := &goBlog{ + cfg: createDefaultTestConfig(t), + httpClient: fc.Client, + } + app.cfg.IndexNow = &configIndexNow{Enabled: true} + _ = app.initConfig() + _ = app.initDatabase(false) + app.initComponents(false) + + // Create http router + app.d, _ = app.buildRouter() + + // Check key + require.NotEmpty(t, app.inKey) + req, _ := http.NewRequest("GET", "http://localhost:8080/"+app.inKey+".txt", nil) + res, err := doHandlerRequest(req, app.d) + require.NoError(t, err) + require.Equal(t, 200, res.StatusCode) + body, _ := io.ReadAll(res.Body) + require.Equal(t, app.inKey, string(body)) + + // Test publish post + _ = app.createPost(&post{ + Section: "posts", + Path: "/testpost", + Published: "2022-01-01", + }) + + // Wait for hooks to run + time.Sleep(300 * time.Millisecond) + + // Check fake http client + require.NotNil(t, fc.req) + require.Equal(t, "https://api.indexnow.org/indexnow?key="+app.inKey+"&url=http%3A%2F%2Flocalhost%3A8080%2Ftestpost", fc.req.URL.String()) +} diff --git a/main.go b/main.go index 6e38bf5..9e20df7 100644 --- a/main.go +++ b/main.go @@ -182,6 +182,7 @@ func (app *goBlog) initComponents(logging bool) { app.initIndieAuth() app.startPostsScheduler() app.initPostsDeleter() + app.initIndexNow() // Log finish if logging { log.Println("Initialized components") diff --git a/persistentCache.go b/persistentCache.go index de365a1..e32b221 100644 --- a/persistentCache.go +++ b/persistentCache.go @@ -2,6 +2,7 @@ package main import ( "database/sql" + "errors" ) func (db *database) cachePersistently(key string, data []byte) error { @@ -11,18 +12,22 @@ func (db *database) cachePersistently(key string, data []byte) error { func (db *database) retrievePersistentCache(key string) (data []byte, err error) { d, err, _ := db.pc.Do(key, func() (interface{}, error) { - if row, err := db.queryRow("select data from persistent_cache where key = @key", sql.Named("key", key)); err == sql.ErrNoRows { - return nil, nil - } else if err != nil { + if row, err := db.queryRow("select data from persistent_cache where key = @key", sql.Named("key", key)); err != nil { return nil, err } else { err = row.Scan(&data) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } return data, err } }) if err != nil { return nil, err } + if d == nil { + return nil, nil + } return d.([]byte), nil }