Browse Source

Full text search

master
Jan-Lukas Else 2 months ago
parent
commit
764c9f7536
13 changed files with 157 additions and 26 deletions
  1. +1
    -1
      .vscode/tasks.json
  2. +1
    -1
      Dockerfile
  3. +10
    -1
      config.go
  4. +25
    -14
      databaseMigrations.go
  5. +13
    -0
      http.go
  6. +21
    -2
      posts.go
  7. +15
    -0
      postsDb.go
  8. +1
    -0
      render.go
  9. +45
    -0
      search.go
  10. +1
    -5
      templates/photos.gohtml
  11. +20
    -0
      templates/search.gohtml
  12. +2
    -1
      templates/strings/de.yaml
  13. +2
    -1
      templates/strings/default.yaml

+ 1
- 1
.vscode/tasks.json View File

@ -4,7 +4,7 @@
{
"label": "Build",
"type": "shell",
"command": "go build"
"command": "go build --tags \"libsqlite3 linux sqlite_fts5\""
}
]
}

+ 1
- 1
Dockerfile View File

@ -4,7 +4,7 @@ ADD *.go /app/
ADD go.mod /app/
ADD go.sum /app/
WORKDIR /app
RUN go build --tags "libsqlite3 linux"
RUN go build --tags "libsqlite3 linux sqlite_fts5"
FROM alpine:3.12
RUN apk add --no-cache sqlite-dev


+ 10
- 1
config.go View File

@ -49,11 +49,12 @@ type configBlog struct {
Title string `mapstructure:"title"`
Description string `mapstructure:"description"`
Pagination int `mapstructure:"pagination"`
DefaultSection string `mapstructure:"defaultsection"`
Sections map[string]*section `mapstructure:"sections"`
Taxonomies []*taxonomy `mapstructure:"taxonomies"`
Menus map[string]*menu `mapstructure:"menus"`
Photos *photos `mapstructure:"photos"`
DefaultSection string `mapstructure:"defaultsection"`
Search *search `mapstructure:"search"`
CustomPages []*customPage `mapstructure:"custompages"`
Telegram *configTelegram `mapstructure:"telegram"`
}
@ -88,6 +89,14 @@ type photos struct {
Description string `mapstructure:"description"`
}
type search struct {
Enabled bool `mapstructure:"enabled"`
Path string `mapstructure:"path"`
Title string `mapstructure:"title"`
Description string `mapstructure:"description"`
Placeholder string `mapstructure:"placeholder"`
}
type customPage struct {
Path string `mapstructure:"path"`
Template string `mapstructure:"template"`


+ 25
- 14
databaseMigrations.go View File

@ -15,17 +15,17 @@ func migrateDb() error {
Name: "00001",
Func: func(tx *sql.Tx) error {
_, err := tx.Exec(`
CREATE TABLE posts (path text not null primary key, content text, published text, updated text, blog text not null, section text);
CREATE TABLE post_parameters (id integer primary key autoincrement, path text not null, parameter text not null, value text);
CREATE INDEX index_pp_path on post_parameters (path);
CREATE TRIGGER AFTER DELETE on posts BEGIN delete from post_parameters where path = old.path; END;
CREATE TABLE indieauthauth (time text not null, code text not null, me text not null, client text not null, redirect text not null, scope text not null);
CREATE TABLE indieauthtoken (time text not null, token text not null, me text not null, client text not null, scope text not null);
CREATE INDEX index_iat_token on indieauthtoken (token);
CREATE TABLE autocert (key text not null primary key, data blob not null, created text not null);
CREATE TABLE activitypub_followers (blog text not null, follower text not null, inbox text not null, primary key (blog, follower));
CREATE TABLE webmentions (id integer primary key autoincrement, source text not null, target text not null, created integer not null, status text not null default "new", title text, content text, author text, type text, UNIQUE(source, target));
CREATE INDEX index_wm_target on webmentions (target);
create table posts (path text not null primary key, content text, published text, updated text, blog text not null, section text);
create table post_parameters (id integer primary key autoincrement, path text not null, parameter text not null, value text);
create index index_pp_path on post_parameters (path);
create trigger after delete on posts begin delete from post_parameters where path = old.path; end;
create table indieauthauth (time text not null, code text not null, me text not null, client text not null, redirect text not null, scope text not null);
create table indieauthtoken (time text not null, token text not null, me text not null, client text not null, scope text not null);
create index index_iat_token on indieauthtoken (token);
create table autocert (key text not null primary key, data blob not null, created text not null);
create table activitypub_followers (blog text not null, follower text not null, inbox text not null, primary key (blog, follower));
create table webmentions (id integer primary key autoincrement, source text not null, target text not null, created integer not null, status text not null default "new", title text, content text, author text, type text, unique(source, target));
create index index_wm_target on webmentions (target);
`)
return err
},
@ -34,7 +34,7 @@ func migrateDb() error {
Name: "00002",
Func: func(tx *sql.Tx) error {
_, err := tx.Exec(`
DROP TABLE autocert;
drop table autocert;
`)
return err
},
@ -43,8 +43,19 @@ func migrateDb() error {
Name: "00003",
Func: func(tx *sql.Tx) error {
_, err := tx.Exec(`
DROP TRIGGER AFTER;
CREATE TRIGGER trigger_posts_delete_pp AFTER DELETE on posts BEGIN delete from post_parameters where path = old.path; END;
drop trigger AFTER;
create trigger trigger_posts_delete_pp after delete on posts begin delete from post_parameters where path = old.path; end;
`)
return err
},
},
&migrator.Migration{
Name: "00004",
Func: func(tx *sql.Tx) error {
_, err := tx.Exec(`
create view view_posts_with_title as select id, path, title, content, published, updated, blog, section from (select p.rowid as id, p.path as path, pp.value as title, content, published, updated, blog, section from posts p left outer join post_parameters pp on p.path = pp.path where pp.parameter = 'title');
create virtual table posts_fts using fts5(path unindexed, title, content, published unindexed, updated unindexed, blog unindexed, section unindexed, content=view_posts_with_title, content_rowid=id);
insert into posts_fts(posts_fts) values ('rebuild');
`)
return err
},


+ 13
- 0
http.go View File

@ -228,6 +228,19 @@ func buildHandler() (http.Handler, error) {
r.With(cacheMiddleware, minifier.Middleware).Get(photoPath+paginationPath, handler)
}
// Search
if blogConfig.Search.Enabled {
searchPath := blogPath + blogConfig.Search.Path
handler := serveSearch(blog, searchPath)
r.With(cacheMiddleware, minifier.Middleware).Get(searchPath, handler)
r.With(cacheMiddleware, minifier.Middleware).Post(searchPath, handler)
searchResultPath := searchPath + "/" + searchPlaceholder
resultHandler := serveSearchResults(blog, searchResultPath)
r.With(cacheMiddleware, minifier.Middleware).Get(searchResultPath, resultHandler)
r.With(cacheMiddleware, minifier.Middleware).Get(searchResultPath+feedPath, resultHandler)
r.With(cacheMiddleware, minifier.Middleware).Get(searchResultPath+paginationPath, resultHandler)
}
// Blog
var mw []func(http.Handler) http.Handler
if appConfig.ActivityPub.Enabled {


+ 21
- 2
posts.go View File

@ -155,6 +155,14 @@ func servePhotos(blog string, path string) func(w http.ResponseWriter, r *http.R
})
}
func serveSearchResults(blog string, path string) func(w http.ResponseWriter, r *http.Request) {
return serveIndex(&indexConfig{
blog: blog,
path: path,
template: templateIndex,
})
}
type indexConfig struct {
blog string
path string
@ -167,6 +175,10 @@ type indexConfig struct {
func serveIndex(ic *indexConfig) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
search := chi.URLParam(r, "search")
if search != "" {
search = searchDecode(search)
}
pageNoString := chi.URLParam(r, "page")
pageNo, _ := strconv.Atoi(pageNoString)
var sections []string
@ -183,6 +195,7 @@ func serveIndex(ic *indexConfig) func(w http.ResponseWriter, r *http.Request) {
taxonomy: ic.tax,
taxonomyValue: ic.taxValue,
parameter: ic.parameter,
search: search,
}}, appConfig.Blogs[ic.blog].Pagination)
p.SetPage(pageNo)
var posts []*post
@ -198,6 +211,8 @@ func serveIndex(ic *indexConfig) func(w http.ResponseWriter, r *http.Request) {
} else if ic.section != nil {
title = ic.section.Title
description = ic.section.Description
} else if search != "" {
title = fmt.Sprintf("%s: %s", appConfig.Blogs[ic.blog].Search.Title, search)
}
// Check if feed
if ft := feedType(chi.URLParam(r, "feed")); ft != noFeed {
@ -217,6 +232,10 @@ func serveIndex(ic *indexConfig) func(w http.ResponseWriter, r *http.Request) {
if len(template) == 0 {
template = templateIndex
}
path := ic.path
if strings.Contains(path, searchPlaceholder) {
path = strings.ReplaceAll(path, searchPlaceholder, searchEncode(search))
}
render(w, template, &renderData{
blogString: ic.blog,
Canonical: appConfig.Server.PublicAddress + r.URL.Path,
@ -227,8 +246,8 @@ func serveIndex(ic *indexConfig) func(w http.ResponseWriter, r *http.Request) {
HasPrev: p.HasPrev(),
HasNext: p.HasNext(),
First: ic.path,
Prev: fmt.Sprintf("%s/page/%d", ic.path, prevPage),
Next: fmt.Sprintf("%s/page/%d", ic.path, nextPage),
Prev: fmt.Sprintf("%s/page/%d", path, prevPage),
Next: fmt.Sprintf("%s/page/%d", path, nextPage),
},
})
}


+ 15
- 0
postsDb.go View File

@ -159,6 +159,7 @@ func (p *post) createOrReplace(new bool) error {
return err
}
finishWritingToDb()
rebuildFTSIndex()
if !postExists {
defer p.postPostHooks()
} else {
@ -176,10 +177,19 @@ func deletePost(path string) error {
return err
}
_, err = appDbExec("delete from posts where path = @path", sql.Named("path", p.Path))
if err != nil {
return err
}
rebuildFTSIndex()
defer p.postDeleteHooks()
return reloadRouter()
}
func rebuildFTSIndex() {
_, _ = appDbExec("insert into posts_fts(posts_fts) values ('rebuild')")
return
}
func postExists(path string) bool {
result := 0
row, err := appDbQueryRow("select exists(select 1 from posts where path = @path)", sql.Named("path", path))
@ -203,6 +213,7 @@ func getPost(path string) (*post, error) {
}
type postsRequestConfig struct {
search string
blog string
path string
limit int
@ -218,6 +229,10 @@ func buildQuery(config *postsRequestConfig) (query string, args []interface{}) {
args = []interface{}{}
defaultSelection := "select p.path as path, coalesce(content, ''), coalesce(published, ''), coalesce(updated, ''), coalesce(blog, ''), coalesce(section, ''), coalesce(parameter, ''), coalesce(value, '') "
postsTable := "posts"
if config.search != "" {
postsTable = "posts_fts(@search)"
args = append(args, sql.Named("search", config.search))
}
if config.blog != "" {
postsTable = "(select * from " + postsTable + " where blog = @blog)"
args = append(args, sql.Named("blog", config.blog))


+ 1
- 0
render.go View File

@ -28,6 +28,7 @@ const templateError = "error"
const templateIndex = "index"
const templateTaxonomy = "taxonomy"
const templatePhotos = "photos"
const templateSearch = "search"
var templates map[string]*template.Template
var templateFunctions template.FuncMap


+ 45
- 0
search.go View File

@ -0,0 +1,45 @@
package main
import (
"encoding/base64"
"net/http"
"net/url"
"strings"
)
const searchPlaceholder = "{search}"
func serveSearch(blog string, path string) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
err := r.ParseForm()
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if q := r.Form.Get("q"); q != "" {
http.Redirect(w, r, path+"/"+searchEncode(q), http.StatusFound)
return
}
render(w, templateSearch, &renderData{
blogString: blog,
Canonical: appConfig.Server.PublicAddress + path,
})
}
}
func searchEncode(search string) string {
return url.PathEscape(strings.ReplaceAll(base64.StdEncoding.EncodeToString([]byte(search)), "/", "_"))
}
func searchDecode(encoded string) string {
encoded, err := url.PathUnescape(encoded)
if err != nil {
return ""
}
encoded = strings.ReplaceAll(encoded, "_", "/")
db, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return ""
}
return string(db)
}

+ 1
- 5
templates/photos.gohtml View File

@ -1,9 +1,5 @@
{{ define "title" }}
{{ if .Blog.Photos.Title }}
<title>{{ .Blog.Photos.Title }} - {{ .Blog.Title }}</title>
{{ else }}
<title>{{ .Blog.Title }}</title>
{{ end }}
<title>{{ with .Blog.Photos.Title }}{{ . }} - {{ end }}{{ .Blog.Title }}</title>
{{ end }}
{{ define "main" }}


+ 20
- 0
templates/search.gohtml View File

@ -0,0 +1,20 @@
{{ define "title" }}
<title>{{ with .Blog.Search.Title }}{{ . }} - {{ end }}{{ .Blog.Title }}</title>
{{ end }}
{{ define "main" }}
<main>
{{ with .Blog.Search.Title }}<h1>{{ . }}</h1>{{ end }}
{{ with .Blog.Search.Description }}{{ md . }}{{ end }}
{{ if (or .Blog.Search.Title .Blog.Search.Description) }}
<hr>
{{ end }}
<form class="fw-form p" method="post">
<input class="fw" type="text" style="margin-bottom: 5px" name="q" {{ with .Blog.Search.Placeholder }}placeholder="{{ . }}"{{ end }} />
<input class="fw" type="submit" value="🔍 {{ string .Blog.Lang "search" }}">
</main>
{{ end }}
{{ define "search" }}
{{ template "base" . }}
{{ end }}

+ 2
- 1
templates/strings/de.yaml View File

@ -9,4 +9,5 @@ translations: "Übersetzungen"
share: "Teilen"
speak: "Lies mir bitte vor."
stopspeak: "Hör auf zu sprechen!"
oldcontent: "⚠️ Dieser Eintrag ist bereits über ein Jahr alt. Er ist möglicherweise nicht mehr aktuell. Meinungen können sich geändert haben."
oldcontent: "⚠️ Dieser Eintrag ist bereits über ein Jahr alt. Er ist möglicherweise nicht mehr aktuell. Meinungen können sich geändert haben."
search: "Suchen"

+ 2
- 1
templates/strings/default.yaml View File

@ -21,4 +21,5 @@ approve: "Approve"
anoncomment: "You can also create an anonymous comment."
interactions: "Interactions"
send: "Send"
interactionslabel: "Have you published a response to this? Paste the URL here."
interactionslabel: "Have you published a response to this? Paste the URL here."
search: "Search"

Loading…
Cancel
Save