mirror of https://github.com/jlelse/GoBlog
Add support for IndexNow
This commit is contained in:
parent
1c3af6d657
commit
48f2ac888b
3
app.go
3
app.go
|
@ -52,6 +52,9 @@ type goBlog struct {
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
// HTTP Routers
|
// HTTP Routers
|
||||||
d http.Handler
|
d http.Handler
|
||||||
|
// IndexNow
|
||||||
|
inKey string
|
||||||
|
inLoad singleflight.Group
|
||||||
// IndieAuth
|
// IndieAuth
|
||||||
ias *indieauth.Server
|
ias *indieauth.Server
|
||||||
// Logs
|
// Logs
|
||||||
|
|
|
@ -24,6 +24,7 @@ type config struct {
|
||||||
Webmention *configWebmention `mapstructure:"webmention"`
|
Webmention *configWebmention `mapstructure:"webmention"`
|
||||||
Notifications *configNotifications `mapstructure:"notifications"`
|
Notifications *configNotifications `mapstructure:"notifications"`
|
||||||
PrivateMode *configPrivateMode `mapstructure:"privateMode"`
|
PrivateMode *configPrivateMode `mapstructure:"privateMode"`
|
||||||
|
IndexNow *configIndexNow `mapstructure:"indexNow"`
|
||||||
EasterEgg *configEasterEgg `mapstructure:"easterEgg"`
|
EasterEgg *configEasterEgg `mapstructure:"easterEgg"`
|
||||||
Debug bool `mapstructure:"debug"`
|
Debug bool `mapstructure:"debug"`
|
||||||
MapTiles *configMapTiles `mapstructure:"mapTiles"`
|
MapTiles *configMapTiles `mapstructure:"mapTiles"`
|
||||||
|
@ -279,6 +280,10 @@ type configPrivateMode struct {
|
||||||
Enabled bool `mapstructure:"enabled"`
|
Enabled bool `mapstructure:"enabled"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type configIndexNow struct {
|
||||||
|
Enabled bool `mapstructure:"enabled"`
|
||||||
|
}
|
||||||
|
|
||||||
type configEasterEgg struct {
|
type configEasterEgg struct {
|
||||||
Enabled bool `mapstructure:"enabled"`
|
Enabled bool `mapstructure:"enabled"`
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,6 +41,10 @@ cache:
|
||||||
privateMode:
|
privateMode:
|
||||||
enabled: true # Enable private mode and only allow access with login
|
enabled: true # Enable private mode and only allow access with login
|
||||||
|
|
||||||
|
# IndexNow (https://www.indexnow.org/index)
|
||||||
|
indexNow:
|
||||||
|
enabled: true # Enable IndexNow integration
|
||||||
|
|
||||||
# User
|
# User
|
||||||
user:
|
user:
|
||||||
name: John Doe # Full name
|
name: John Doe # Full name
|
||||||
|
|
4
go.mod
4
go.mod
|
@ -56,7 +56,7 @@ require (
|
||||||
github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38
|
github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38
|
||||||
github.com/yuin/goldmark-highlighting v0.0.0-20210516132338-9216f9c5aa01
|
github.com/yuin/goldmark-highlighting v0.0.0-20210516132338-9216f9c5aa01
|
||||||
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce
|
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce
|
||||||
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
|
golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba
|
||||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||||
golang.org/x/text v0.3.7
|
golang.org/x/text v0.3.7
|
||||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
|
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
|
||||||
|
@ -130,7 +130,7 @@ require (
|
||||||
go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect
|
go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect
|
||||||
golang.org/x/mod v0.5.1 // indirect
|
golang.org/x/mod v0.5.1 // indirect
|
||||||
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
|
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
|
||||||
golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect
|
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
|
||||||
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
|
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
|
||||||
golang.org/x/tools v0.1.8 // indirect
|
golang.org/x/tools v0.1.8 // indirect
|
||||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||||
|
|
8
go.sum
8
go.sum
|
@ -553,8 +553,8 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx
|
||||||
golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||||
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||||
golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||||
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
|
golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba h1:6u6sik+bn/y7vILcYkK3iwTBWN7WtBvB0+SZswQnbf8=
|
||||||
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||||
|
@ -631,8 +631,8 @@ golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk=
|
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
|
||||||
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
|
||||||
|
|
7
http.go
7
http.go
|
@ -205,6 +205,13 @@ func (a *goBlog) buildRouter() (http.Handler, error) {
|
||||||
// Sitemap
|
// Sitemap
|
||||||
r.With(a.privateModeHandler, cacheLoggedIn, a.cacheMiddleware).Get(sitemapPath, a.serveSitemap)
|
r.With(a.privateModeHandler, cacheLoggedIn, a.cacheMiddleware).Get(sitemapPath, a.serveSitemap)
|
||||||
|
|
||||||
|
// IndexNow
|
||||||
|
if a.indexNowEnabled() {
|
||||||
|
if inkey := a.indexNowKey(); inkey != "" {
|
||||||
|
r.With(cacheLoggedIn, a.cacheMiddleware).Get("/"+inkey+".txt", a.serveIndexNow)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Robots.txt
|
// Robots.txt
|
||||||
r.With(cacheLoggedIn, a.cacheMiddleware).Get(robotsTXTPath, a.serveRobotsTXT)
|
r.With(cacheLoggedIn, a.cacheMiddleware).Get(robotsTXTPath, a.serveRobotsTXT)
|
||||||
|
|
||||||
|
|
|
@ -2,7 +2,6 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"golang.org/x/crypto/acme/autocert"
|
"golang.org/x/crypto/acme/autocert"
|
||||||
|
@ -27,7 +26,7 @@ func (c *httpsCache) Get(_ context.Context, key string) ([]byte, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
d, err := c.db.retrievePersistentCache("https_" + key)
|
d, err := c.db.retrievePersistentCache("https_" + key)
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if d == nil && err == nil {
|
||||||
return nil, autocert.ErrCacheMiss
|
return nil, autocert.ErrCacheMiss
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|
|
@ -0,0 +1,97 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"github.com/carlmjohnson/requests"
|
||||||
|
"github.com/thoas/go-funk"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Implement support for the IndexNow protocol
|
||||||
|
// https://www.indexnow.org/documentation
|
||||||
|
|
||||||
|
func (a *goBlog) initIndexNow() {
|
||||||
|
if !a.indexNowEnabled() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Add hooks
|
||||||
|
hook := func(p *post) {
|
||||||
|
// Check if post is published
|
||||||
|
if !p.isPublishedSectionPost() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Send IndexNow request
|
||||||
|
a.indexNow(a.fullPostURL(p))
|
||||||
|
}
|
||||||
|
a.pPostHooks = append(a.pPostHooks, hook)
|
||||||
|
a.pUpdateHooks = append(a.pUpdateHooks, hook)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *goBlog) indexNowEnabled() bool {
|
||||||
|
// Check if private mode is enabled
|
||||||
|
if a.isPrivate() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// Check if IndexNow is disabled
|
||||||
|
if inc := a.cfg.IndexNow; inc == nil || !inc.Enabled {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *goBlog) serveIndexNow(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Write([]byte(a.indexNowKey()))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *goBlog) indexNow(url string) {
|
||||||
|
if !a.indexNowEnabled() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
key := a.indexNowKey()
|
||||||
|
if key == "" {
|
||||||
|
log.Println("Skipping IndexNow")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err := requests.URL("https://api.indexnow.org/indexnow").
|
||||||
|
Client(a.httpClient).
|
||||||
|
UserAgent(appUserAgent).
|
||||||
|
Param("url", url).
|
||||||
|
Param("key", key).
|
||||||
|
Fetch(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
log.Println("Sending IndexNow request failed:", err.Error())
|
||||||
|
return
|
||||||
|
} else {
|
||||||
|
log.Println("IndexNow request sent for", url)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *goBlog) indexNowKey() string {
|
||||||
|
res, _, _ := a.inLoad.Do("", func() (interface{}, error) {
|
||||||
|
// Check if already loaded
|
||||||
|
if a.inKey != "" {
|
||||||
|
return a.inKey, nil
|
||||||
|
}
|
||||||
|
// Try to load key from database
|
||||||
|
keyBytes, err := a.db.retrievePersistentCache("indexnowkey")
|
||||||
|
if err != nil {
|
||||||
|
log.Println("Failed to retrieve cached IndexNow key:", err.Error())
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if keyBytes == nil {
|
||||||
|
// Generate 128 character key with hexadecimal characters
|
||||||
|
keyBytes = []byte(funk.RandomString(128, []rune("0123456789abcdef")))
|
||||||
|
// Store key in database
|
||||||
|
err = a.db.cachePersistently("indexnowkey", keyBytes)
|
||||||
|
if err != nil {
|
||||||
|
log.Println("Failed to cache IndexNow key:", err.Error())
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.inKey = string(keyBytes)
|
||||||
|
return a.inKey, nil
|
||||||
|
})
|
||||||
|
return res.(string)
|
||||||
|
}
|
|
@ -0,0 +1,50 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func Test_indexNow(t *testing.T) {
|
||||||
|
fc := newFakeHttpClient()
|
||||||
|
fc.setFakeResponse(200, "OK")
|
||||||
|
|
||||||
|
app := &goBlog{
|
||||||
|
cfg: createDefaultTestConfig(t),
|
||||||
|
httpClient: fc.Client,
|
||||||
|
}
|
||||||
|
app.cfg.IndexNow = &configIndexNow{Enabled: true}
|
||||||
|
_ = app.initConfig()
|
||||||
|
_ = app.initDatabase(false)
|
||||||
|
app.initComponents(false)
|
||||||
|
|
||||||
|
// Create http router
|
||||||
|
app.d, _ = app.buildRouter()
|
||||||
|
|
||||||
|
// Check key
|
||||||
|
require.NotEmpty(t, app.inKey)
|
||||||
|
req, _ := http.NewRequest("GET", "http://localhost:8080/"+app.inKey+".txt", nil)
|
||||||
|
res, err := doHandlerRequest(req, app.d)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, 200, res.StatusCode)
|
||||||
|
body, _ := io.ReadAll(res.Body)
|
||||||
|
require.Equal(t, app.inKey, string(body))
|
||||||
|
|
||||||
|
// Test publish post
|
||||||
|
_ = app.createPost(&post{
|
||||||
|
Section: "posts",
|
||||||
|
Path: "/testpost",
|
||||||
|
Published: "2022-01-01",
|
||||||
|
})
|
||||||
|
|
||||||
|
// Wait for hooks to run
|
||||||
|
time.Sleep(300 * time.Millisecond)
|
||||||
|
|
||||||
|
// Check fake http client
|
||||||
|
require.NotNil(t, fc.req)
|
||||||
|
require.Equal(t, "https://api.indexnow.org/indexnow?key="+app.inKey+"&url=http%3A%2F%2Flocalhost%3A8080%2Ftestpost", fc.req.URL.String())
|
||||||
|
}
|
1
main.go
1
main.go
|
@ -182,6 +182,7 @@ func (app *goBlog) initComponents(logging bool) {
|
||||||
app.initIndieAuth()
|
app.initIndieAuth()
|
||||||
app.startPostsScheduler()
|
app.startPostsScheduler()
|
||||||
app.initPostsDeleter()
|
app.initPostsDeleter()
|
||||||
|
app.initIndexNow()
|
||||||
// Log finish
|
// Log finish
|
||||||
if logging {
|
if logging {
|
||||||
log.Println("Initialized components")
|
log.Println("Initialized components")
|
||||||
|
|
|
@ -2,6 +2,7 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (db *database) cachePersistently(key string, data []byte) error {
|
func (db *database) cachePersistently(key string, data []byte) error {
|
||||||
|
@ -11,18 +12,22 @@ func (db *database) cachePersistently(key string, data []byte) error {
|
||||||
|
|
||||||
func (db *database) retrievePersistentCache(key string) (data []byte, err error) {
|
func (db *database) retrievePersistentCache(key string) (data []byte, err error) {
|
||||||
d, err, _ := db.pc.Do(key, func() (interface{}, error) {
|
d, err, _ := db.pc.Do(key, func() (interface{}, error) {
|
||||||
if row, err := db.queryRow("select data from persistent_cache where key = @key", sql.Named("key", key)); err == sql.ErrNoRows {
|
if row, err := db.queryRow("select data from persistent_cache where key = @key", sql.Named("key", key)); err != nil {
|
||||||
return nil, nil
|
|
||||||
} else if err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
} else {
|
} else {
|
||||||
err = row.Scan(&data)
|
err = row.Scan(&data)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
return data, err
|
return data, err
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if d == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
return d.([]byte), nil
|
return d.([]byte), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue