Full text search

.vscode/tasks.json

@@ -4,7 +4,7 @@
       {
         "label": "Build",
         "type": "shell",
-        "command": "go build"
+        "command": "go build --tags \"libsqlite3 linux sqlite_fts5\""
       }
     ]
   }

Dockerfile

@@ -4,7 +4,7 @@ ADD *.go /app/
 ADD go.mod /app/
 ADD go.sum /app/
 WORKDIR /app
-RUN go build --tags "libsqlite3 linux"
+RUN go build --tags "libsqlite3 linux sqlite_fts5"
 
 FROM alpine:3.12
 RUN apk add --no-cache sqlite-dev

config.go

@@ -49,11 +49,12 @@ type configBlog struct {
 	Title          string              `mapstructure:"title"`
 	Description    string              `mapstructure:"description"`
 	Pagination     int                 `mapstructure:"pagination"`
+	DefaultSection string              `mapstructure:"defaultsection"`
 	Sections       map[string]*section `mapstructure:"sections"`
 	Taxonomies     []*taxonomy         `mapstructure:"taxonomies"`
 	Menus          map[string]*menu    `mapstructure:"menus"`
 	Photos         *photos             `mapstructure:"photos"`
-	DefaultSection string              `mapstructure:"defaultsection"`
+	Search         *search             `mapstructure:"search"`
 	CustomPages    []*customPage       `mapstructure:"custompages"`
 	Telegram       *configTelegram     `mapstructure:"telegram"`
 }
@@ -88,6 +89,14 @@ type photos struct {
 	Description string `mapstructure:"description"`
 }
 
+type search struct {
+	Enabled     bool   `mapstructure:"enabled"`
+	Path        string `mapstructure:"path"`
+	Title       string `mapstructure:"title"`
+	Description string `mapstructure:"description"`
+	Placeholder string `mapstructure:"placeholder"`
+}
+
 type customPage struct {
 	Path     string       `mapstructure:"path"`
 	Template string       `mapstructure:"template"`

databaseMigrations.go

@@ -15,17 +15,17 @@ func migrateDb() error {
 				Name: "00001",
 				Func: func(tx *sql.Tx) error {
 					_, err := tx.Exec(`
-					CREATE TABLE posts (path text not null primary key, content text, published text, updated text, blog text not null, section text);
+					create table posts (path text not null primary key, content text, published text, updated text, blog text not null, section text);
-					CREATE TABLE post_parameters (id integer primary key autoincrement, path text not null, parameter text not null, value text);
+					create table post_parameters (id integer primary key autoincrement, path text not null, parameter text not null, value text);
-					CREATE INDEX index_pp_path on post_parameters (path);
+					create index index_pp_path on post_parameters (path);
-					CREATE TRIGGER AFTER DELETE on posts BEGIN delete from post_parameters where path = old.path; END;
+					create trigger after delete on posts begin delete from post_parameters where path = old.path; end;
-					CREATE TABLE indieauthauth (time text not null, code text not null, me text not null, client text not null, redirect text not null, scope text not null);
+					create table indieauthauth (time text not null, code text not null, me text not null, client text not null, redirect text not null, scope text not null);
-					CREATE TABLE indieauthtoken (time text not null, token text not null, me text not null, client text not null, scope text not null);
+					create table indieauthtoken (time text not null, token text not null, me text not null, client text not null, scope text not null);
-					CREATE INDEX index_iat_token on indieauthtoken (token);
+					create index index_iat_token on indieauthtoken (token);
-					CREATE TABLE autocert (key text not null primary key, data blob not null, created text not null);
+					create table autocert (key text not null primary key, data blob not null, created text not null);
-					CREATE TABLE activitypub_followers (blog text not null, follower text not null, inbox text not null, primary key (blog, follower));
+					create table activitypub_followers (blog text not null, follower text not null, inbox text not null, primary key (blog, follower));
-					CREATE TABLE webmentions (id integer primary key autoincrement, source text not null, target text not null, created integer not null, status text not null default "new", title text, content text, author text, type text, UNIQUE(source, target));
+					create table webmentions (id integer primary key autoincrement, source text not null, target text not null, created integer not null, status text not null default "new", title text, content text, author text, type text, unique(source, target));
-					CREATE INDEX index_wm_target on webmentions (target);
+					create index index_wm_target on webmentions (target);
 					`)
 					return err
 				},
@@ -34,7 +34,7 @@ func migrateDb() error {
 				Name: "00002",
 				Func: func(tx *sql.Tx) error {
 					_, err := tx.Exec(`
-					DROP TABLE autocert;
+					drop table autocert;
 					`)
 					return err
 				},
@@ -43,8 +43,19 @@ func migrateDb() error {
 				Name: "00003",
 				Func: func(tx *sql.Tx) error {
 					_, err := tx.Exec(`
-					DROP TRIGGER AFTER;
+					drop trigger AFTER;
-					CREATE TRIGGER trigger_posts_delete_pp AFTER DELETE on posts BEGIN delete from post_parameters where path = old.path; END;
+					create trigger trigger_posts_delete_pp after delete on posts begin delete from post_parameters where path = old.path; end;
+					`)
+					return err
+				},
+			},
+			&migrator.Migration{
+				Name: "00004",
+				Func: func(tx *sql.Tx) error {
+					_, err := tx.Exec(`
+					create view view_posts_with_title as select id, path, title, content, published, updated, blog, section from (select p.rowid as id, p.path as path, pp.value as title, content, published, updated, blog, section from posts p left outer join post_parameters pp on p.path = pp.path where pp.parameter = 'title');
+					create virtual table posts_fts using fts5(path unindexed, title, content, published unindexed, updated unindexed, blog unindexed, section unindexed, content=view_posts_with_title, content_rowid=id);
+					insert into posts_fts(posts_fts) values ('rebuild');
 					`)
 					return err
 				},

http.go

@@ -228,6 +228,19 @@ func buildHandler() (http.Handler, error) {
 			r.With(cacheMiddleware, minifier.Middleware).Get(photoPath+paginationPath, handler)
 		}
 
+		// Search
+		if blogConfig.Search.Enabled {
+			searchPath := blogPath + blogConfig.Search.Path
+			handler := serveSearch(blog, searchPath)
+			r.With(cacheMiddleware, minifier.Middleware).Get(searchPath, handler)
+			r.With(cacheMiddleware, minifier.Middleware).Post(searchPath, handler)
+			searchResultPath := searchPath + "/" + searchPlaceholder
+			resultHandler := serveSearchResults(blog, searchResultPath)
+			r.With(cacheMiddleware, minifier.Middleware).Get(searchResultPath, resultHandler)
+			r.With(cacheMiddleware, minifier.Middleware).Get(searchResultPath+feedPath, resultHandler)
+			r.With(cacheMiddleware, minifier.Middleware).Get(searchResultPath+paginationPath, resultHandler)
+		}
+
 		// Blog
 		var mw []func(http.Handler) http.Handler
 		if appConfig.ActivityPub.Enabled {

posts.go

@@ -155,6 +155,14 @@ func servePhotos(blog string, path string) func(w http.ResponseWriter, r *http.R
 	})
 }
 
+func serveSearchResults(blog string, path string) func(w http.ResponseWriter, r *http.Request) {
+	return serveIndex(&indexConfig{
+		blog:     blog,
+		path:     path,
+		template: templateIndex,
+	})
+}
+
 type indexConfig struct {
 	blog      string
 	path      string
@@ -167,6 +175,10 @@ type indexConfig struct {
 
 func serveIndex(ic *indexConfig) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
+		search := chi.URLParam(r, "search")
+		if search != "" {
+			search = searchDecode(search)
+		}
 		pageNoString := chi.URLParam(r, "page")
 		pageNo, _ := strconv.Atoi(pageNoString)
 		var sections []string
@@ -183,6 +195,7 @@ func serveIndex(ic *indexConfig) func(w http.ResponseWriter, r *http.Request) {
 			taxonomy:      ic.tax,
 			taxonomyValue: ic.taxValue,
 			parameter:     ic.parameter,
+			search:        search,
 		}}, appConfig.Blogs[ic.blog].Pagination)
 		p.SetPage(pageNo)
 		var posts []*post
@@ -198,6 +211,8 @@ func serveIndex(ic *indexConfig) func(w http.ResponseWriter, r *http.Request) {
 		} else if ic.section != nil {
 			title = ic.section.Title
 			description = ic.section.Description
+		} else if search != "" {
+			title = fmt.Sprintf("%s: %s", appConfig.Blogs[ic.blog].Search.Title, search)
 		}
 		// Check if feed
 		if ft := feedType(chi.URLParam(r, "feed")); ft != noFeed {
@@ -217,6 +232,10 @@ func serveIndex(ic *indexConfig) func(w http.ResponseWriter, r *http.Request) {
 		if len(template) == 0 {
 			template = templateIndex
 		}
+		path := ic.path
+		if strings.Contains(path, searchPlaceholder) {
+			path = strings.ReplaceAll(path, searchPlaceholder, searchEncode(search))
+		}
 		render(w, template, &renderData{
 			blogString: ic.blog,
 			Canonical:  appConfig.Server.PublicAddress + r.URL.Path,
@@ -227,8 +246,8 @@ func serveIndex(ic *indexConfig) func(w http.ResponseWriter, r *http.Request) {
 				HasPrev:     p.HasPrev(),
 				HasNext:     p.HasNext(),
 				First:       ic.path,
-				Prev:        fmt.Sprintf("%s/page/%d", ic.path, prevPage),
+				Prev:        fmt.Sprintf("%s/page/%d", path, prevPage),
-				Next:        fmt.Sprintf("%s/page/%d", ic.path, nextPage),
+				Next:        fmt.Sprintf("%s/page/%d", path, nextPage),
 			},
 		})
 	}

postsDb.go

@@ -159,6 +159,7 @@ func (p *post) createOrReplace(new bool) error {
 		return err
 	}
 	finishWritingToDb()
+	rebuildFTSIndex()
 	if !postExists {
 		defer p.postPostHooks()
 	} else {
@@ -176,10 +177,19 @@ func deletePost(path string) error {
 		return err
 	}
 	_, err = appDbExec("delete from posts where path = @path", sql.Named("path", p.Path))
+	if err != nil {
+		return err
+	}
+	rebuildFTSIndex()
 	defer p.postDeleteHooks()
 	return reloadRouter()
 }
 
+func rebuildFTSIndex() {
+	_, _ = appDbExec("insert into posts_fts(posts_fts) values ('rebuild')")
+	return
+}
+
 func postExists(path string) bool {
 	result := 0
 	row, err := appDbQueryRow("select exists(select 1 from posts where path = @path)", sql.Named("path", path))
@@ -203,6 +213,7 @@ func getPost(path string) (*post, error) {
 }
 
 type postsRequestConfig struct {
+	search         string
 	blog           string
 	path           string
 	limit          int
@@ -218,6 +229,10 @@ func buildQuery(config *postsRequestConfig) (query string, args []interface{}) {
 	args = []interface{}{}
 	defaultSelection := "select p.path as path, coalesce(content, ''), coalesce(published, ''), coalesce(updated, ''), coalesce(blog, ''), coalesce(section, ''), coalesce(parameter, ''), coalesce(value, '') "
 	postsTable := "posts"
+	if config.search != "" {
+		postsTable = "posts_fts(@search)"
+		args = append(args, sql.Named("search", config.search))
+	}
 	if config.blog != "" {
 		postsTable = "(select * from " + postsTable + " where blog = @blog)"
 		args = append(args, sql.Named("blog", config.blog))

render.go

@@ -28,6 +28,7 @@ const templateError = "error"
 const templateIndex = "index"
 const templateTaxonomy = "taxonomy"
 const templatePhotos = "photos"
+const templateSearch = "search"
 
 var templates map[string]*template.Template
 var templateFunctions template.FuncMap

search.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"encoding/base64"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+const searchPlaceholder = "{search}"
+
+func serveSearch(blog string, path string) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		err := r.ParseForm()
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		if q := r.Form.Get("q"); q != "" {
+			http.Redirect(w, r, path+"/"+searchEncode(q), http.StatusFound)
+			return
+		}
+		render(w, templateSearch, &renderData{
+			blogString: blog,
+			Canonical:  appConfig.Server.PublicAddress + path,
+		})
+	}
+}
+
+func searchEncode(search string) string {
+	return url.PathEscape(strings.ReplaceAll(base64.StdEncoding.EncodeToString([]byte(search)), "/", "_"))
+}
+
+func searchDecode(encoded string) string {
+	encoded, err := url.PathUnescape(encoded)
+	if err != nil {
+		return ""
+	}
+	encoded = strings.ReplaceAll(encoded, "_", "/")
+	db, err := base64.StdEncoding.DecodeString(encoded)
+	if err != nil {
+		return ""
+	}
+	return string(db)
+}

templates/photos.gohtml

@@ -1,9 +1,5 @@
 {{ define "title" }}
-    {{ if .Blog.Photos.Title }}
+  <title>{{ with .Blog.Photos.Title }}{{ . }} - {{ end }}{{ .Blog.Title }}</title>
-      <title>{{ .Blog.Photos.Title }} - {{ .Blog.Title }}</title>
-    {{ else }}
-      <title>{{ .Blog.Title }}</title>
-    {{ end }}
 {{ end }}
 
 {{ define "main" }}

templates/search.gohtml

@@ -0,0 +1,20 @@
+{{ define "title" }}
+  <title>{{ with .Blog.Search.Title }}{{ . }} - {{ end }}{{ .Blog.Title }}</title>
+{{ end }}
+
+{{ define "main" }}
+  <main>
+    {{ with .Blog.Search.Title }}<h1>{{ . }}</h1>{{ end }}
+    {{ with .Blog.Search.Description }}{{ md . }}{{ end }}
+    {{ if (or .Blog.Search.Title .Blog.Search.Description) }}
+      <hr>
+    {{ end }}
+    <form class="fw-form p" method="post">
+    <input class="fw" type="text" style="margin-bottom: 5px" name="q" {{ with .Blog.Search.Placeholder }}placeholder="{{ . }}"{{ end }} />
+    <input class="fw" type="submit" value="🔍 {{ string .Blog.Lang "search" }}">
+  </main>
+{{ end }}
+
+{{ define "search" }}
+    {{ template "base" . }}
+{{ end }}

templates/strings/de.yaml

@@ -10,3 +10,4 @@ share: "Teilen"
 speak: "Lies mir bitte vor."
 stopspeak: "Hör auf zu sprechen!"
 oldcontent: "⚠️ Dieser Eintrag ist bereits über ein Jahr alt. Er ist möglicherweise nicht mehr aktuell. Meinungen können sich geändert haben."
+search: "Suchen"

templates/strings/default.yaml

@@ -22,3 +22,4 @@ anoncomment: "You can also create an anonymous comment."
 interactions: "Interactions"
 send: "Send"
 interactionslabel: "Have you published a response to this? Paste the URL here."
+search: "Search"