mirror of https://github.com/jlelse/GoBlog
Add support for IndexNow
This commit is contained in:
parent
1c3af6d657
commit
48f2ac888b
3
app.go
3
app.go
|
@ -52,6 +52,9 @@ type goBlog struct {
|
|||
httpClient *http.Client
|
||||
// HTTP Routers
|
||||
d http.Handler
|
||||
// IndexNow
|
||||
inKey string
|
||||
inLoad singleflight.Group
|
||||
// IndieAuth
|
||||
ias *indieauth.Server
|
||||
// Logs
|
||||
|
|
|
@ -24,6 +24,7 @@ type config struct {
|
|||
Webmention *configWebmention `mapstructure:"webmention"`
|
||||
Notifications *configNotifications `mapstructure:"notifications"`
|
||||
PrivateMode *configPrivateMode `mapstructure:"privateMode"`
|
||||
IndexNow *configIndexNow `mapstructure:"indexNow"`
|
||||
EasterEgg *configEasterEgg `mapstructure:"easterEgg"`
|
||||
Debug bool `mapstructure:"debug"`
|
||||
MapTiles *configMapTiles `mapstructure:"mapTiles"`
|
||||
|
@ -279,6 +280,10 @@ type configPrivateMode struct {
|
|||
Enabled bool `mapstructure:"enabled"`
|
||||
}
|
||||
|
||||
type configIndexNow struct {
|
||||
Enabled bool `mapstructure:"enabled"`
|
||||
}
|
||||
|
||||
type configEasterEgg struct {
|
||||
Enabled bool `mapstructure:"enabled"`
|
||||
}
|
||||
|
|
|
@ -41,6 +41,10 @@ cache:
|
|||
privateMode:
|
||||
enabled: true # Enable private mode and only allow access with login
|
||||
|
||||
# IndexNow (https://www.indexnow.org/index)
|
||||
indexNow:
|
||||
enabled: true # Enable IndexNow integration
|
||||
|
||||
# User
|
||||
user:
|
||||
name: John Doe # Full name
|
||||
|
|
4
go.mod
4
go.mod
|
@ -56,7 +56,7 @@ require (
|
|||
github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38
|
||||
github.com/yuin/goldmark-highlighting v0.0.0-20210516132338-9216f9c5aa01
|
||||
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce
|
||||
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
|
||||
golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
golang.org/x/text v0.3.7
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
|
||||
|
@ -130,7 +130,7 @@ require (
|
|||
go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect
|
||||
golang.org/x/mod v0.5.1 // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
|
||||
golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
|
||||
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
|
||||
golang.org/x/tools v0.1.8 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||
|
|
8
go.sum
8
go.sum
|
@ -553,8 +553,8 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx
|
|||
golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
|
||||
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba h1:6u6sik+bn/y7vILcYkK3iwTBWN7WtBvB0+SZswQnbf8=
|
||||
golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
@ -631,8 +631,8 @@ golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk=
|
||||
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
|
||||
|
|
7
http.go
7
http.go
|
@ -205,6 +205,13 @@ func (a *goBlog) buildRouter() (http.Handler, error) {
|
|||
// Sitemap
|
||||
r.With(a.privateModeHandler, cacheLoggedIn, a.cacheMiddleware).Get(sitemapPath, a.serveSitemap)
|
||||
|
||||
// IndexNow
|
||||
if a.indexNowEnabled() {
|
||||
if inkey := a.indexNowKey(); inkey != "" {
|
||||
r.With(cacheLoggedIn, a.cacheMiddleware).Get("/"+inkey+".txt", a.serveIndexNow)
|
||||
}
|
||||
}
|
||||
|
||||
// Robots.txt
|
||||
r.With(cacheLoggedIn, a.cacheMiddleware).Get(robotsTXTPath, a.serveRobotsTXT)
|
||||
|
||||
|
|
|
@ -2,7 +2,6 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
|
||||
"golang.org/x/crypto/acme/autocert"
|
||||
|
@ -27,7 +26,7 @@ func (c *httpsCache) Get(_ context.Context, key string) ([]byte, error) {
|
|||
return nil, err
|
||||
}
|
||||
d, err := c.db.retrievePersistentCache("https_" + key)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
if d == nil && err == nil {
|
||||
return nil, autocert.ErrCacheMiss
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
|
|
|
@ -0,0 +1,97 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/carlmjohnson/requests"
|
||||
"github.com/thoas/go-funk"
|
||||
)
|
||||
|
||||
// Implement support for the IndexNow protocol
|
||||
// https://www.indexnow.org/documentation
|
||||
|
||||
func (a *goBlog) initIndexNow() {
|
||||
if !a.indexNowEnabled() {
|
||||
return
|
||||
}
|
||||
// Add hooks
|
||||
hook := func(p *post) {
|
||||
// Check if post is published
|
||||
if !p.isPublishedSectionPost() {
|
||||
return
|
||||
}
|
||||
// Send IndexNow request
|
||||
a.indexNow(a.fullPostURL(p))
|
||||
}
|
||||
a.pPostHooks = append(a.pPostHooks, hook)
|
||||
a.pUpdateHooks = append(a.pUpdateHooks, hook)
|
||||
}
|
||||
|
||||
func (a *goBlog) indexNowEnabled() bool {
|
||||
// Check if private mode is enabled
|
||||
if a.isPrivate() {
|
||||
return false
|
||||
}
|
||||
// Check if IndexNow is disabled
|
||||
if inc := a.cfg.IndexNow; inc == nil || !inc.Enabled {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (a *goBlog) serveIndexNow(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(a.indexNowKey()))
|
||||
}
|
||||
|
||||
func (a *goBlog) indexNow(url string) {
|
||||
if !a.indexNowEnabled() {
|
||||
return
|
||||
}
|
||||
key := a.indexNowKey()
|
||||
if key == "" {
|
||||
log.Println("Skipping IndexNow")
|
||||
return
|
||||
}
|
||||
err := requests.URL("https://api.indexnow.org/indexnow").
|
||||
Client(a.httpClient).
|
||||
UserAgent(appUserAgent).
|
||||
Param("url", url).
|
||||
Param("key", key).
|
||||
Fetch(context.Background())
|
||||
if err != nil {
|
||||
log.Println("Sending IndexNow request failed:", err.Error())
|
||||
return
|
||||
} else {
|
||||
log.Println("IndexNow request sent for", url)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *goBlog) indexNowKey() string {
|
||||
res, _, _ := a.inLoad.Do("", func() (interface{}, error) {
|
||||
// Check if already loaded
|
||||
if a.inKey != "" {
|
||||
return a.inKey, nil
|
||||
}
|
||||
// Try to load key from database
|
||||
keyBytes, err := a.db.retrievePersistentCache("indexnowkey")
|
||||
if err != nil {
|
||||
log.Println("Failed to retrieve cached IndexNow key:", err.Error())
|
||||
return "", err
|
||||
}
|
||||
if keyBytes == nil {
|
||||
// Generate 128 character key with hexadecimal characters
|
||||
keyBytes = []byte(funk.RandomString(128, []rune("0123456789abcdef")))
|
||||
// Store key in database
|
||||
err = a.db.cachePersistently("indexnowkey", keyBytes)
|
||||
if err != nil {
|
||||
log.Println("Failed to cache IndexNow key:", err.Error())
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
a.inKey = string(keyBytes)
|
||||
return a.inKey, nil
|
||||
})
|
||||
return res.(string)
|
||||
}
|
|
@ -0,0 +1,50 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_indexNow(t *testing.T) {
|
||||
fc := newFakeHttpClient()
|
||||
fc.setFakeResponse(200, "OK")
|
||||
|
||||
app := &goBlog{
|
||||
cfg: createDefaultTestConfig(t),
|
||||
httpClient: fc.Client,
|
||||
}
|
||||
app.cfg.IndexNow = &configIndexNow{Enabled: true}
|
||||
_ = app.initConfig()
|
||||
_ = app.initDatabase(false)
|
||||
app.initComponents(false)
|
||||
|
||||
// Create http router
|
||||
app.d, _ = app.buildRouter()
|
||||
|
||||
// Check key
|
||||
require.NotEmpty(t, app.inKey)
|
||||
req, _ := http.NewRequest("GET", "http://localhost:8080/"+app.inKey+".txt", nil)
|
||||
res, err := doHandlerRequest(req, app.d)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, res.StatusCode)
|
||||
body, _ := io.ReadAll(res.Body)
|
||||
require.Equal(t, app.inKey, string(body))
|
||||
|
||||
// Test publish post
|
||||
_ = app.createPost(&post{
|
||||
Section: "posts",
|
||||
Path: "/testpost",
|
||||
Published: "2022-01-01",
|
||||
})
|
||||
|
||||
// Wait for hooks to run
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
|
||||
// Check fake http client
|
||||
require.NotNil(t, fc.req)
|
||||
require.Equal(t, "https://api.indexnow.org/indexnow?key="+app.inKey+"&url=http%3A%2F%2Flocalhost%3A8080%2Ftestpost", fc.req.URL.String())
|
||||
}
|
1
main.go
1
main.go
|
@ -182,6 +182,7 @@ func (app *goBlog) initComponents(logging bool) {
|
|||
app.initIndieAuth()
|
||||
app.startPostsScheduler()
|
||||
app.initPostsDeleter()
|
||||
app.initIndexNow()
|
||||
// Log finish
|
||||
if logging {
|
||||
log.Println("Initialized components")
|
||||
|
|
|
@ -2,6 +2,7 @@ package main
|
|||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
)
|
||||
|
||||
func (db *database) cachePersistently(key string, data []byte) error {
|
||||
|
@ -11,18 +12,22 @@ func (db *database) cachePersistently(key string, data []byte) error {
|
|||
|
||||
func (db *database) retrievePersistentCache(key string) (data []byte, err error) {
|
||||
d, err, _ := db.pc.Do(key, func() (interface{}, error) {
|
||||
if row, err := db.queryRow("select data from persistent_cache where key = @key", sql.Named("key", key)); err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
} else if err != nil {
|
||||
if row, err := db.queryRow("select data from persistent_cache where key = @key", sql.Named("key", key)); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
err = row.Scan(&data)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return data, err
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if d == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return d.([]byte), nil
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue