Add some features, improve database handling & performance, robots.txt and more

This commit is contained in:
Jan-Lukas Else 2020-11-09 16:40:12 +01:00
parent 85f010b895
commit 5b9ac19cb8
15 changed files with 408 additions and 317 deletions

View File

@ -5,6 +5,7 @@ import (
"context"
"crypto/rsa"
"crypto/x509"
"database/sql"
"encoding/pem"
"errors"
"fmt"
@ -182,7 +183,7 @@ func apGetRemoteActor(iri string) (*asPerson, error) {
}
func apGetAllFollowers(blog string) (map[string]string, error) {
rows, err := appDb.Query("select follower, inbox from activitypub_followers where blog = ?", blog)
rows, err := appDbQuery("select follower, inbox from activitypub_followers where blog = @blog", sql.Named("blog", blog))
if err != nil {
return nil, err
}
@ -199,29 +200,23 @@ func apGetAllFollowers(blog string) (map[string]string, error) {
}
func apAddFollower(blog, follower, inbox string) error {
startWritingToDb()
defer finishWritingToDb()
_, err := appDb.Exec("insert or replace into activitypub_followers (blog, follower, inbox) values (?, ?, ?)", blog, follower, inbox)
if err != nil {
_, err := appDbExec("insert or replace into activitypub_followers (blog, follower, inbox) values (@blog, @follower, @inbox)", sql.Named("blog", blog), sql.Named("follower", follower), sql.Named("inbox", inbox))
return err
}
return nil
}
func apRemoveFollower(blog, follower string) error {
startWritingToDb()
defer finishWritingToDb()
_, err := appDb.Exec("delete from activitypub_followers where blog = ? and follower = ?", blog, follower)
if err != nil {
_, err := appDbExec("delete from activitypub_followers where blog = @blog and follower = @follower", sql.Named("blog", blog), sql.Named("follower", follower))
return err
}
return nil
}
func apPost(p *post) {
func (p *post) apPost() {
if !appConfig.ActivityPub.Enabled {
return
}
if p.Published == "" || p.firstParameter("section") == "" {
// No section, don't post
return
}
n := p.toASNote()
createActivity := make(map[string]interface{})
createActivity["@context"] = asContext
@ -233,11 +228,25 @@ func apPost(p *post) {
apSendToAllFollowers(p.Blog, createActivity)
}
func apUpdate(p *post) {
// TODO
func (p *post) apUpdate() {
if !appConfig.ActivityPub.Enabled {
return
}
if p.Published == "" || p.firstParameter("section") == "" {
// No section, don't post
return
}
n := p.toASNote()
updateActivity := make(map[string]interface{})
updateActivity["@context"] = asContext
updateActivity["actor"] = appConfig.Blogs[p.Blog].apIri()
updateActivity["id"] = appConfig.Server.PublicAddress + p.Path
updateActivity["type"] = "Update"
updateActivity["object"] = n
apSendToAllFollowers(p.Blog, updateActivity)
}
func apDelete(p *post) {
func (p *post) apDelete() {
if !appConfig.ActivityPub.Enabled {
return
}

View File

@ -8,25 +8,15 @@ import (
"golang.org/x/crypto/acme/autocert"
)
type autocertCache struct {
db *sql.DB
getQuery string
putQuery string
deleteQuery string
}
func newAutocertCache() (*autocertCache, error) {
return &autocertCache{
db: appDb,
getQuery: "select data from autocert where key = ?",
putQuery: "insert or replace into autocert (key, data, created) values (?, ?, ?)",
deleteQuery: "delete from autocert where key = ?",
}, nil
}
type autocertCache struct{}
func (c *autocertCache) Get(ctx context.Context, key string) ([]byte, error) {
var data []byte
err := c.db.QueryRowContext(ctx, c.getQuery, key).Scan(&data)
row, err := appDbQueryRow("select data from autocert where key = @key", sql.Named("key", key))
if err != nil {
return nil, err
}
err = row.Scan(&data)
if err == sql.ErrNoRows {
return nil, autocert.ErrCacheMiss
}
@ -36,13 +26,13 @@ func (c *autocertCache) Get(ctx context.Context, key string) ([]byte, error) {
func (c *autocertCache) Put(ctx context.Context, key string, data []byte) error {
startWritingToDb()
defer finishWritingToDb()
_, err := c.db.ExecContext(ctx, c.putQuery, key, data, time.Now().String())
_, err := appDbExec("insert or replace into autocert (key, data, created) values (@key, @data, @created)", sql.Named("key", key), sql.Named("data", data), sql.Named("created", time.Now().String()))
return err
}
func (c *autocertCache) Delete(ctx context.Context, key string) error {
startWritingToDb()
defer finishWritingToDb()
_, err := c.db.ExecContext(ctx, c.deleteQuery, key)
_, err := appDbExec("delete from autocert where key = @key", sql.Named("key", key))
return err
}

View File

@ -1,6 +1,7 @@
package main
import (
"io/ioutil"
"net/http"
"net/http/httptest"
"sync"
@ -102,11 +103,13 @@ func getCache(key string, next http.Handler, r *http.Request) *cacheItem {
recorder := httptest.NewRecorder()
next.ServeHTTP(recorder, r)
// Cache values from recorder
result := recorder.Result()
body, _ := ioutil.ReadAll(result.Body)
item = &cacheItem{
creationTime: time.Now().Unix(),
code: recorder.Code,
header: recorder.Header(),
body: recorder.Body.Bytes(),
code: result.StatusCode,
header: result.Header,
body: body,
}
// Save cache
cacheMutex.Lock()

View File

@ -116,6 +116,7 @@ type configHooks struct {
PreStart []string `mapstructure:"prestart"`
// Can use template
PostPost []string `mapstructure:"postpost"`
PostUpdate []string `mapstructure:"postupdate"`
PostDelete []string `mapstructure:"postdelete"`
}

View File

@ -7,8 +7,12 @@ import (
_ "github.com/mattn/go-sqlite3"
)
var appDb *sql.DB
var appDbWriteMutex = &sync.Mutex{}
var (
appDb *sql.DB
appDbWriteMutex = &sync.Mutex{}
dbStatementCache = map[string]*sql.Stmt{}
dbStatementCacheMutex = &sync.RWMutex{}
)
func initDatabase() (err error) {
appDb, err = sql.Open("sqlite3", appConfig.Db.File+"?cache=shared&mode=rwc&_journal_mode=WAL")
@ -32,7 +36,51 @@ func closeDb() error {
}
func vacuumDb() {
_, _ = appDbExec("VACUUM;")
}
func prepareAppDbStatement(query string) (*sql.Stmt, error) {
stmt, err, _ := cacheGroup.Do(query, func() (interface{}, error) {
dbStatementCacheMutex.RLock()
stmt, ok := dbStatementCache[query]
dbStatementCacheMutex.RUnlock()
if ok && stmt != nil {
return stmt, nil
}
stmt, err := appDb.Prepare(query)
if err != nil {
return nil, err
}
dbStatementCacheMutex.Lock()
dbStatementCache[query] = stmt
dbStatementCacheMutex.Unlock()
return stmt, nil
})
return stmt.(*sql.Stmt), err
}
func appDbExec(query string, args ...interface{}) (sql.Result, error) {
stmt, err := prepareAppDbStatement(query)
if err != nil {
return nil, err
}
startWritingToDb()
defer finishWritingToDb()
_, _ = appDb.Exec("VACUUM;")
return stmt.Exec(args...)
}
func appDbQuery(query string, args ...interface{}) (*sql.Rows, error) {
stmt, err := prepareAppDbStatement(query)
if err != nil {
return nil, err
}
return stmt.Query(args...)
}
func appDbQueryRow(query string, args ...interface{}) (*sql.Row, error) {
stmt, err := prepareAppDbStatement(query)
if err != nil {
return nil, err
}
return stmt.QueryRow(args...), nil
}

View File

@ -18,6 +18,7 @@ func migrateDb() error {
CREATE TABLE posts (path text not null primary key, content text, published text, updated text, blog text not null, section text);
CREATE TABLE post_parameters (id integer primary key autoincrement, path text not null, parameter text not null, value text);
CREATE INDEX index_pp_path on post_parameters (path);
CREATE TRIGGER AFTER DELETE on posts BEGIN delete from post_parameters where path = old.path; END;
CREATE TABLE redirects (fromPath text not null, toPath text not null, primary key (fromPath, toPath));
CREATE TABLE indieauthauth (time text not null, code text not null, me text not null, client text not null, redirect text not null, scope text not null);
CREATE TABLE indieauthtoken (time text not null, token text not null, me text not null, client text not null, scope text not null);

View File

@ -17,31 +17,50 @@ func preStartHooks() {
}
}
func postPostHooks(path string) {
func (p *post) postPostHooks() {
for _, cmdTmplString := range appConfig.Hooks.PostPost {
go func(path, cmdTmplString string) {
executeTemplateCommand("post-post", cmdTmplString, &hookTemplateData{
URL: appConfig.Server.PublicAddress + path,
go func(p *post, cmdTmplString string) {
executeTemplateCommand("post-post", cmdTmplString, map[string]interface{}{
"URL": appConfig.Server.PublicAddress + p.Path,
"Post": p,
})
}(path, cmdTmplString)
}(p, cmdTmplString)
}
go p.apPost()
go p.sendWebmentions()
}
func postDeleteHooks(path string) {
for _, cmdTmplString := range appConfig.Hooks.PostDelete {
go func(path, cmdTmplString string) {
executeTemplateCommand("post-delete", cmdTmplString, &hookTemplateData{
URL: appConfig.Server.PublicAddress + path,
func (p *post) postUpdateHooks() {
for _, cmdTmplString := range appConfig.Hooks.PostUpdate {
go func(p *post, cmdTmplString string) {
executeTemplateCommand("post-update", cmdTmplString, map[string]interface{}{
"URL": appConfig.Server.PublicAddress + p.Path,
"Post": p,
})
}(path, cmdTmplString)
}(p, cmdTmplString)
}
go p.apUpdate()
go p.sendWebmentions()
}
func (p *post) postDeleteHooks() {
for _, cmdTmplString := range appConfig.Hooks.PostDelete {
go func(p *post, cmdTmplString string) {
executeTemplateCommand("post-delete", cmdTmplString, map[string]interface{}{
"URL": appConfig.Server.PublicAddress + p.Path,
"Post": p,
})
}(p, cmdTmplString)
}
go p.apDelete()
go p.sendWebmentions()
}
type hookTemplateData struct {
URL string
}
func executeTemplateCommand(hookType string, tmpl string, data *hookTemplateData) {
func executeTemplateCommand(hookType string, tmpl string, data map[string]interface{}) {
cmdTmpl, err := template.New("cmd").Parse(tmpl)
if err != nil {
log.Println("Failed to parse cmd template:", err.Error())

11
http.go
View File

@ -63,14 +63,10 @@ func startServer() (err error) {
}
localAddress := ":" + strconv.Itoa(appConfig.Server.Port)
if appConfig.Server.PublicHTTPS {
cache, err := newAutocertCache()
if err != nil {
return err
}
certManager := autocert.Manager{
Prompt: autocert.AcceptTOS,
HostPolicy: autocert.HostWhitelist(appConfig.Server.Domain),
Cache: cache,
Cache: &autocertCache{},
Email: appConfig.Server.LetsEncryptMail,
}
tlsConfig := certManager.TLSConfig()
@ -94,6 +90,7 @@ func reloadRouter() error {
if err != nil {
return err
}
purgeCache()
d.swapHandler(h)
return nil
}
@ -104,6 +101,7 @@ func buildHandler() (http.Handler, error) {
if appConfig.Server.Logging {
r.Use(logMiddleware)
}
// r.Use(middleware.Logger)
r.Use(middleware.Recoverer)
r.Use(middleware.Compress(flate.DefaultCompression))
r.Use(middleware.RedirectSlashes)
@ -265,6 +263,9 @@ func buildHandler() (http.Handler, error) {
// Sitemap
r.With(cacheMiddleware, minifier.Middleware).Get(sitemapPath, serveSitemap)
// Robots.txt - doesn't need cache, because it's too simple
r.Get("/robots.txt", serveRobotsTXT)
// Check redirects, then serve 404
r.With(checkRegexRedirects, cacheMiddleware, minifier.Middleware).NotFound(serve404)

View File

@ -222,16 +222,17 @@ func indieAuthToken(w http.ResponseWriter, r *http.Request) {
}
func (data *indieAuthData) saveAuthorization() (err error) {
startWritingToDb()
defer finishWritingToDb()
_, err = appDb.Exec("insert into indieauthauth (time, code, me, client, redirect, scope) values (?, ?, ?, ?, ?, ?)", data.time.Unix(), data.code, data.Me, data.ClientID, data.RedirectURI, strings.Join(data.Scopes, " "))
_, err = appDbExec("insert into indieauthauth (time, code, me, client, redirect, scope) values (?, ?, ?, ?, ?, ?)", data.time.Unix(), data.code, data.Me, data.ClientID, data.RedirectURI, strings.Join(data.Scopes, " "))
return
}
func (data *indieAuthData) verifyAuthorization(authentication bool) (valid bool, err error) {
// code valid for 600 seconds
if !authentication {
row := appDb.QueryRow("select code, me, client, redirect, scope from indieauthauth where time >= ? and code = ? and me = ? and client = ? and redirect = ?", time.Now().Unix()-600, data.code, data.Me, data.ClientID, data.RedirectURI)
row, err := appDbQueryRow("select code, me, client, redirect, scope from indieauthauth where time >= ? and code = ? and me = ? and client = ? and redirect = ?", time.Now().Unix()-600, data.code, data.Me, data.ClientID, data.RedirectURI)
if err != nil {
return false, err
}
scope := ""
err = row.Scan(&data.code, &data.Me, &data.ClientID, &data.RedirectURI, &scope)
if err == sql.ErrNoRows {
@ -243,7 +244,10 @@ func (data *indieAuthData) verifyAuthorization(authentication bool) (valid bool,
data.Scopes = strings.Split(scope, " ")
}
} else {
row := appDb.QueryRow("select code, me, client, redirect from indieauthauth where time >= ? and code = ? and client = ? and redirect = ?", time.Now().Unix()-600, data.code, data.ClientID, data.RedirectURI)
row, err := appDbQueryRow("select code, me, client, redirect from indieauthauth where time >= ? and code = ? and client = ? and redirect = ?", time.Now().Unix()-600, data.code, data.ClientID, data.RedirectURI)
if err != nil {
return false, err
}
err = row.Scan(&data.code, &data.Me, &data.ClientID, &data.RedirectURI)
if err == sql.ErrNoRows {
return false, nil
@ -252,24 +256,23 @@ func (data *indieAuthData) verifyAuthorization(authentication bool) (valid bool,
}
}
valid = true
startWritingToDb()
defer finishWritingToDb()
_, err = appDb.Exec("delete from indieauthauth where code = ? or time < ?", data.code, time.Now().Unix()-600)
_, err = appDbExec("delete from indieauthauth where code = ? or time < ?", data.code, time.Now().Unix()-600)
data.code = ""
return
}
func (data *indieAuthData) saveToken() (err error) {
startWritingToDb()
defer finishWritingToDb()
_, err = appDb.Exec("insert into indieauthtoken (time, token, me, client, scope) values (?, ?, ?, ?, ?)", data.time.Unix(), data.token, data.Me, data.ClientID, strings.Join(data.Scopes, " "))
_, err = appDbExec("insert into indieauthtoken (time, token, me, client, scope) values (?, ?, ?, ?, ?)", data.time.Unix(), data.token, data.Me, data.ClientID, strings.Join(data.Scopes, " "))
return
}
func verifyIndieAuthToken(token string) (data *indieAuthData, err error) {
token = strings.ReplaceAll(token, "Bearer ", "")
data = &indieAuthData{}
row := appDb.QueryRow("select time, token, me, client, scope from indieauthtoken where token = ?", token)
row, err := appDbQueryRow("select time, token, me, client, scope from indieauthtoken where token = ?", token)
if err != nil {
return nil, err
}
timeString := ""
scope := ""
err = row.Scan(&timeString, &data.token, &data.Me, &data.ClientID, &scope)
@ -289,8 +292,6 @@ func revokeIndieAuthToken(token string) {
if token == "" {
return
}
startWritingToDb()
defer finishWritingToDb()
_, _ = appDb.Exec("delete from indieauthtoken where token=?", token)
_, _ = appDbExec("delete from indieauthtoken where token=?", token)
return
}

128
posts.go
View File

@ -233,131 +233,3 @@ func serveIndex(ic *indexConfig) func(w http.ResponseWriter, r *http.Request) {
})
}
}
func getPost(path string) (*post, error) {
posts, err := getPosts(&postsRequestConfig{path: path})
if err != nil {
return nil, err
} else if len(posts) == 0 {
return nil, errPostNotFound
}
return posts[0], nil
}
type postsRequestConfig struct {
blog string
path string
limit int
offset int
sections []string
taxonomy *taxonomy
taxonomyValue string
parameter string
parameterValue string
}
func buildQuery(config *postsRequestConfig) (query string, params []interface{}) {
defaultSelection := "select p.path as path, coalesce(content, ''), coalesce(published, ''), coalesce(updated, ''), coalesce(blog, ''), coalesce(section, ''), coalesce(parameter, ''), coalesce(value, '') "
postsTable := "posts"
if config.blog != "" {
postsTable = "(select * from " + postsTable + " where blog = '" + config.blog + "')"
}
if config.parameter != "" {
if config.parameterValue != "" {
postsTable = "(select distinct p.* from " + postsTable + " p left outer join post_parameters pp on p.path = pp.path where pp.parameter = '" + config.parameter + "' and pp.value = '" + config.parameterValue + "')"
} else {
postsTable = "(select distinct p.* from " + postsTable + " p left outer join post_parameters pp on p.path = pp.path where pp.parameter = '" + config.parameter + "' and length(coalesce(pp.value, '')) > 1)"
}
}
if config.taxonomy != nil && len(config.taxonomyValue) > 0 {
postsTable = "(select distinct p.* from " + postsTable + " p left outer join post_parameters pp on p.path = pp.path where pp.parameter = '" + config.taxonomy.Name + "' and lower(pp.value) = lower('" + config.taxonomyValue + "'))"
}
if len(config.sections) > 0 {
postsTable = "(select * from " + postsTable + " where"
for i, section := range config.sections {
if i > 0 {
postsTable += " or"
}
postsTable += " section='" + section + "'"
}
postsTable += ")"
}
defaultTables := " from " + postsTable + " p left outer join post_parameters pp on p.path = pp.path "
defaultSorting := " order by p.published desc "
if config.path != "" {
query = defaultSelection + defaultTables + " where p.path=?" + defaultSorting
params = []interface{}{config.path}
} else if config.limit != 0 || config.offset != 0 {
query = defaultSelection + " from (select * from " + postsTable + " p " + defaultSorting + " limit ? offset ?) p left outer join post_parameters pp on p.path = pp.path "
params = []interface{}{config.limit, config.offset}
} else {
query = defaultSelection + defaultTables + defaultSorting
}
return
}
func getPosts(config *postsRequestConfig) (posts []*post, err error) {
query, queryParams := buildQuery(config)
rows, err := appDb.Query(query, queryParams...)
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
paths := make(map[string]int)
for rows.Next() {
p := &post{}
var parameterName, parameterValue string
err = rows.Scan(&p.Path, &p.Content, &p.Published, &p.Updated, &p.Blog, &p.Section, &parameterName, &parameterValue)
if err != nil {
return nil, err
}
if paths[p.Path] == 0 {
index := len(posts)
paths[p.Path] = index + 1
p.Parameters = make(map[string][]string)
posts = append(posts, p)
}
if parameterName != "" && posts != nil {
posts[paths[p.Path]-1].Parameters[parameterName] = append(posts[paths[p.Path]-1].Parameters[parameterName], parameterValue)
}
}
return posts, nil
}
func countPosts(config *postsRequestConfig) (count int, err error) {
query, params := buildQuery(config)
query = "select count(distinct path) from (" + query + ")"
row := appDb.QueryRow(query, params...)
err = row.Scan(&count)
return
}
func allPostPaths() ([]string, error) {
var postPaths []string
rows, err := appDb.Query("select path from posts")
if err != nil {
return nil, err
}
for rows.Next() {
var path string
_ = rows.Scan(&path)
postPaths = append(postPaths, path)
}
return postPaths, nil
}
func allTaxonomyValues(blog string, taxonomy string) ([]string, error) {
var values []string
rows, err := appDb.Query("select distinct pp.value from posts p left outer join post_parameters pp on p.path = pp.path where pp.parameter = ? and length(coalesce(pp.value, '')) > 1 and blog = ?", taxonomy, blog)
if err != nil {
return nil, err
}
for rows.Next() {
var value string
_ = rows.Scan(&value)
values = append(values, value)
}
return values, nil
}

View File

@ -2,6 +2,7 @@ package main
import (
"bytes"
"database/sql"
"errors"
"fmt"
"strings"
@ -108,22 +109,33 @@ func (p *post) createOrReplace(new bool) error {
return err
}
startWritingToDb()
postExists := postExists(p.Path)
if postExists && new {
finishWritingToDb()
return errors.New("post already exists at given path")
}
tx, err := appDb.Begin()
if err != nil {
finishWritingToDb()
return err
}
sqlCommand := "insert"
if !new {
sqlCommand = "insert or replace"
}
_, err = tx.Exec(sqlCommand+" into posts (path, content, published, updated, blog, section) values (?, ?, ?, ?, ?, ?)", p.Path, p.Content, p.Published, p.Updated, p.Blog, p.Section)
if postExists {
_, err := tx.Exec("delete from posts where path = @path", sql.Named("path", p.Path))
if err != nil {
_ = tx.Rollback()
finishWritingToDb()
return err
}
_, err = tx.Exec("delete from post_parameters where path=?", p.Path)
}
_, err = tx.Exec(
"insert into posts (path, content, published, updated, blog, section) values (@path, @content, @published, @updated, @blog, @section)",
sql.Named("path", p.Path), sql.Named("content", p.Content), sql.Named("published", p.Published), sql.Named("updated", p.Updated), sql.Named("blog", p.Blog), sql.Named("section", p.Section))
if err != nil {
_ = tx.Rollback()
finishWritingToDb()
return err
}
ppStmt, err := tx.Prepare("insert into post_parameters (path, parameter, value) values (@path, @parameter, @value)")
if err != nil {
_ = tx.Rollback()
finishWritingToDb()
@ -132,7 +144,7 @@ func (p *post) createOrReplace(new bool) error {
for param, value := range p.Parameters {
for _, value := range value {
if value != "" {
_, err = tx.Exec("insert into post_parameters (path, parameter, value) values (?, ?, ?)", p.Path, param, value)
_, err := ppStmt.Exec(sql.Named("path", p.Path), sql.Named("parameter", param), sql.Named("value", value))
if err != nil {
_ = tx.Rollback()
finishWritingToDb()
@ -147,11 +159,11 @@ func (p *post) createOrReplace(new bool) error {
return err
}
finishWritingToDb()
purgeCache()
defer func(p *post) {
postPostHooks(p.Path)
go apPost(p)
}(p)
if !postExists {
defer p.postPostHooks()
} else {
defer p.postUpdateHooks()
}
return reloadRouter()
}
@ -163,34 +175,158 @@ func deletePost(path string) error {
if err != nil {
return err
}
startWritingToDb()
tx, err := appDb.Begin()
if err != nil {
finishWritingToDb()
return err
}
_, err = tx.Exec("delete from posts where path=?", p.Path)
if err != nil {
_ = tx.Rollback()
finishWritingToDb()
return err
}
_, err = tx.Exec("delete from post_parameters where path=?", p.Path)
if err != nil {
_ = tx.Rollback()
finishWritingToDb()
return err
}
err = tx.Commit()
if err != nil {
finishWritingToDb()
return err
}
finishWritingToDb()
purgeCache()
defer func(p *post) {
postDeleteHooks(p.Path)
apDelete(p)
}(p)
_, err = appDbExec("delete from posts where path = @path", sql.Named("path", p.Path))
defer p.postDeleteHooks()
return reloadRouter()
}
func postExists(path string) bool {
result := 0
row, err := appDbQueryRow("select exists(select 1 from posts where path = @path)", sql.Named("path", path))
if err != nil {
return false
}
if err = row.Scan(&result); err != nil {
return false
}
return result == 1
}
func getPost(path string) (*post, error) {
posts, err := getPosts(&postsRequestConfig{path: path})
if err != nil {
return nil, err
} else if len(posts) == 0 {
return nil, errPostNotFound
}
return posts[0], nil
}
type postsRequestConfig struct {
blog string
path string
limit int
offset int
sections []string
taxonomy *taxonomy
taxonomyValue string
parameter string
parameterValue string
}
func buildQuery(config *postsRequestConfig) (query string, args []interface{}) {
args = []interface{}{}
defaultSelection := "select p.path as path, coalesce(content, ''), coalesce(published, ''), coalesce(updated, ''), coalesce(blog, ''), coalesce(section, ''), coalesce(parameter, ''), coalesce(value, '') "
postsTable := "posts"
if config.blog != "" {
postsTable = "(select * from " + postsTable + " where blog = @blog)"
args = append(args, sql.Named("blog", config.blog))
}
if config.parameter != "" {
postsTable = "(select distinct p.* from " + postsTable + " p left outer join post_parameters pp on p.path = pp.path where pp.parameter = @param "
args = append(args, sql.Named("param", config.parameter))
if config.parameterValue != "" {
postsTable += "and pp.value = @paramval)"
args = append(args, sql.Named("paramval", config.parameterValue))
} else {
postsTable += "and length(coalesce(pp.value, '')) > 1)"
}
}
if config.taxonomy != nil && len(config.taxonomyValue) > 0 {
postsTable = "(select distinct p.* from " + postsTable + " p left outer join post_parameters pp on p.path = pp.path where pp.parameter = @taxname and lower(pp.value) = lower(@taxval))"
args = append(args, sql.Named("taxname", config.taxonomy.Name), sql.Named("taxval", config.taxonomyValue))
}
if len(config.sections) > 0 {
postsTable = "(select * from " + postsTable + " where"
for i, section := range config.sections {
if i > 0 {
postsTable += " or"
}
named := fmt.Sprintf("section%v", i)
postsTable += fmt.Sprintf(" section = @%v", named)
args = append(args, sql.Named(named, section))
}
postsTable += ")"
}
defaultTables := " from " + postsTable + " p left outer join post_parameters pp on p.path = pp.path "
defaultSorting := " order by p.published desc "
if config.path != "" {
query = defaultSelection + defaultTables + " where p.path = @path" + defaultSorting
args = append(args, sql.Named("path", config.path))
} else if config.limit != 0 || config.offset != 0 {
query = defaultSelection + " from (select * from " + postsTable + " p " + defaultSorting + " limit @limit offset @offset) p left outer join post_parameters pp on p.path = pp.path "
args = append(args, sql.Named("limit", config.limit), sql.Named("offset", config.offset))
} else {
query = defaultSelection + defaultTables + defaultSorting
}
return
}
func getPosts(config *postsRequestConfig) (posts []*post, err error) {
query, queryParams := buildQuery(config)
rows, err := appDbQuery(query, queryParams...)
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
paths := make(map[string]int)
for rows.Next() {
p := &post{}
var parameterName, parameterValue string
err = rows.Scan(&p.Path, &p.Content, &p.Published, &p.Updated, &p.Blog, &p.Section, &parameterName, &parameterValue)
if err != nil {
return nil, err
}
if paths[p.Path] == 0 {
index := len(posts)
paths[p.Path] = index + 1
p.Parameters = make(map[string][]string)
posts = append(posts, p)
}
if parameterName != "" && posts != nil {
posts[paths[p.Path]-1].Parameters[parameterName] = append(posts[paths[p.Path]-1].Parameters[parameterName], parameterValue)
}
}
return posts, nil
}
func countPosts(config *postsRequestConfig) (count int, err error) {
query, params := buildQuery(config)
query = "select count(distinct path) from (" + query + ")"
row, err := appDbQueryRow(query, params...)
if err != nil {
return
}
err = row.Scan(&count)
return
}
func allPostPaths() ([]string, error) {
var postPaths []string
rows, err := appDbQuery("select path from posts")
if err != nil {
return nil, err
}
for rows.Next() {
var path string
_ = rows.Scan(&path)
postPaths = append(postPaths, path)
}
return postPaths, nil
}
func allTaxonomyValues(blog string, taxonomy string) ([]string, error) {
var values []string
rows, err := appDbQuery("select distinct pp.value from posts p left outer join post_parameters pp on p.path = pp.path where pp.parameter = @tax and length(coalesce(pp.value, '')) > 1 and blog = @blog", sql.Named("tax", taxonomy), sql.Named("blog", blog))
if err != nil {
return nil, err
}
for rows.Next() {
var value string
_ = rows.Scan(&value)
values = append(values, value)
}
return values, nil
}

View File

@ -28,8 +28,11 @@ func serveRedirect(w http.ResponseWriter, r *http.Request) {
func getRedirect(fromPath string) (string, error) {
var toPath string
row := appDb.QueryRow("with recursive f (i, fp, tp) as (select 1, fromPath, toPath from redirects where fromPath = ? union all select f.i + 1, r.fromPath, r.toPath from redirects as r join f on f.tp = r.fromPath) select tp from f order by i desc limit 1", fromPath)
err := row.Scan(&toPath)
row, err := appDbQueryRow("with recursive f (i, fp, tp) as (select 1, fromPath, toPath from redirects where fromPath = ? union all select f.i + 1, r.fromPath, r.toPath from redirects as r join f on f.tp = r.fromPath) select tp from f order by i desc limit 1", fromPath)
if err != nil {
return "", err
}
err = row.Scan(&toPath)
if err == sql.ErrNoRows {
return "", errRedirectNotFound
} else if err != nil {
@ -40,7 +43,7 @@ func getRedirect(fromPath string) (string, error) {
func allRedirectPaths() ([]string, error) {
var redirectPaths []string
rows, err := appDb.Query("select fromPath from redirects")
rows, err := appDbQuery("select fromPath from redirects")
if err != nil {
return nil, err
}
@ -61,23 +64,9 @@ func createOrReplaceRedirect(from, to string) error {
return nil
}
from = strings.TrimSuffix(from, "/")
startWritingToDb()
tx, err := appDb.Begin()
_, err := appDbExec("insert or replace into redirects (fromPath, toPath) values (?, ?)", from, to)
if err != nil {
finishWritingToDb()
return err
}
_, err = tx.Exec("insert or replace into redirects (fromPath, toPath) values (?, ?)", from, to)
if err != nil {
_ = tx.Rollback()
finishWritingToDb()
return err
}
err = tx.Commit()
if err != nil {
finishWritingToDb()
return err
}
finishWritingToDb()
return reloadRouter()
}

11
robotstxt.go Normal file
View File

@ -0,0 +1,11 @@
package main
import (
"fmt"
"net/http"
)
func serveRobotsTXT(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte(fmt.Sprintf("User-agent: *\nSitemap: %v", appConfig.Server.PublicAddress+sitemapPath)))
}

View File

@ -0,0 +1,23 @@
{{ define "title" }}{{ end }}
{{ define "main" }}
<main class=h-entry>
<article>
<data class="u-url hide" value="{{ absolute .Data.Path }}"></data>
{{ with title .Data }}<h1 class=p-name>{{ . }}</h1>{{ end }}
{{ include "postmeta" . }}
{{ if .Data.Content }}
<div class=e-content>
{{ content .Data }}
{{ with p .Data "link" }}
<p><a class="u-bookmark-of" href="{{ . }}" target="_blank" rel="noopener">{{ . }}</a></p>
{{ end }}
</div>
{{ end }}
</article>
</main>
{{ end }}
{{ define "postbasic" }}
{{ template "base" . }}
{{ end }}

View File

@ -122,7 +122,11 @@ func webmentionAdminApprove(w http.ResponseWriter, r *http.Request) {
func webmentionExists(source, target string) bool {
result := 0
if err := appDb.QueryRow("select exists(select 1 from webmentions where source = ? and target = ?)", source, target).Scan(&result); err != nil {
row, err := appDbQueryRow("select exists(select 1 from webmentions where source = ? and target = ?)", source, target)
if err != nil {
return false
}
if err = row.Scan(&result); err != nil {
return false
}
return result == 1
@ -131,10 +135,13 @@ func webmentionExists(source, target string) bool {
func verifyNextWebmention() error {
m := &mention{}
oldStatus := ""
if err := appDb.QueryRow("select id, source, target, status from webmentions where (status = ? or status = ?) limit 1", webmentionStatusNew, webmentionStatusRenew).Scan(&m.ID, &m.Source, &m.Target, &oldStatus); err != nil {
if err == sql.ErrNoRows {
return nil
row, err := appDbQueryRow("select id, source, target, status from webmentions where (status = ? or status = ?) limit 1", webmentionStatusNew, webmentionStatusRenew)
if err != nil {
return err
}
if err := row.Scan(&m.ID, &m.Source, &m.Target, &oldStatus); err == sql.ErrNoRows {
return nil
} else if err != nil {
return err
}
wmm := &wmd.Mention{
@ -150,9 +157,7 @@ func verifyNextWebmention() error {
if len(wmm.Content) > 500 {
wmm.Content = wmm.Content[0:497] + "…"
}
startWritingToDb()
defer finishWritingToDb()
_, err := appDb.Exec("update webmentions set status = ?, title = ?, type = ?, content = ?, author = ? where id = ?", webmentionStatusVerified, wmm.Title, wmm.Type, wmm.Content, wmm.AuthorName, m.ID)
_, err = appDbExec("update webmentions set status = ?, title = ?, type = ?, content = ?, author = ? where id = ?", webmentionStatusVerified, wmm.Title, wmm.Type, wmm.Content, wmm.AuthorName, m.ID)
if oldStatus == string(webmentionStatusNew) {
sendNotification(fmt.Sprintf("New webmention from %s to %s", m.Source, m.Target))
}
@ -161,28 +166,20 @@ func verifyNextWebmention() error {
func createWebmention(source, target string) (err error) {
if webmentionExists(source, target) {
startWritingToDb()
defer finishWritingToDb()
_, err = appDb.Exec("update webmentions set status = ? where source = ? and target = ?", webmentionStatusRenew, source, target)
_, err = appDbExec("update webmentions set status = ? where source = ? and target = ?", webmentionStatusRenew, source, target)
} else {
startWritingToDb()
defer finishWritingToDb()
_, err = appDb.Exec("insert into webmentions (source, target, created) values (?, ?, ?)", source, target, time.Now().Unix())
_, err = appDbExec("insert into webmentions (source, target, created) values (?, ?, ?)", source, target, time.Now().Unix())
}
return err
}
func deleteWebmention(id int) error {
startWritingToDb()
defer finishWritingToDb()
_, err := appDb.Exec("delete from webmentions where id = ?", id)
_, err := appDbExec("delete from webmentions where id = ?", id)
return err
}
func approveWebmention(id int) error {
startWritingToDb()
defer finishWritingToDb()
_, err := appDb.Exec("update webmentions set status = ? where id = ?", webmentionStatusApproved, id)
_, err := appDbExec("update webmentions set status = ? where id = ?", webmentionStatusApproved, id)
return err
}
@ -195,19 +192,21 @@ func getWebmentions(config *webmentionsRequestConfig) ([]*mention, error) {
mentions := []*mention{}
var rows *sql.Rows
var err error
filter := "where 1 = 1 "
args := []interface{}{}
filter := ""
if config != nil {
if config.target != "" {
filter += "and target = ? "
args = append(args, config.target)
}
if config.status != "" {
filter += "and status = ? "
args = append(args, config.status)
if config.target != "" && config.status != "" {
filter = "where target = @target and status = @status"
args = append(args, sql.Named("target", config.target), sql.Named("status", config.status))
} else if config.target != "" {
filter = "where target = @target"
args = append(args, sql.Named("target", config.target))
} else if config.status != "" {
filter = "where status = @status"
args = append(args, sql.Named("status", config.status))
}
}
rows, err = appDb.Query("select id, source, target, created, title, content, author, type from webmentions "+filter+"order by created desc", args...)
rows, err = appDbQuery("select id, source, target, created, title, content, author, type from webmentions "+filter+" order by created desc", args...)
if err != nil {
return nil, err
}
@ -222,37 +221,25 @@ func getWebmentions(config *webmentionsRequestConfig) ([]*mention, error) {
return mentions, nil
}
// TODO: Integrate
func sendWebmentions(url string, prefixBlocks ...string) error {
req, err := http.NewRequest(http.MethodGet, url, nil)
func (p *post) sendWebmentions() error {
url := appConfig.Server.PublicAddress + p.Path
recorder := httptest.NewRecorder()
// Render basic post data
render(recorder, "postbasic", &renderData{
blogString: p.Blog,
Data: p,
})
discovered, err := webmention.DiscoverLinksFromReader(recorder.Result().Body, url, ".h-entry")
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
discovered, err := webmention.DiscoverLinksFromReader(resp.Body, url, ".h-entry")
_ = resp.Body.Close()
if err != nil {
return err
}
var filtered []string
allowed := func(link string) bool {
for _, block := range prefixBlocks {
if strings.HasPrefix(link, block) {
return false
}
}
return true
}
for _, link := range discovered {
if allowed(link) {
filtered = append(filtered, link)
}
}
client := webmention.New(nil)
for _, link := range filtered {
for _, link := range discovered {
if strings.HasPrefix(link, appConfig.Server.PublicAddress) {
// Save mention directly
createWebmention(url, link)
continue
}
endpoint, err := client.DiscoverEndpoint(link)
if err != nil || len(endpoint) < 1 {
continue