jlelse
/
GoBlog
mirror of https://github.com/jlelse/GoBlog
1
Fork 0
Code Packages Releases Activity

More and improved tests and new method to count characters for stats

This commit is contained in:
Jan-Lukas Else 2021-06-15 17:36:41 +02:00
parent 4508b8569f
commit 32339e5c41
12 changed files with 282 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
blogstats.go
View File

@ -61,7 +61,7 @@ func (db *database) getBlogStats(blog string) (data map[string]interface{}, err
query, params := buildPostsQuery(prq)
query = "select path, mdtext(content) as content, published, substr(published, 1, 4) as year, substr(published, 6, 2) as month from (" + query + ")"
postCount := "coalesce(count(distinct path), 0) as postcount"
charCount := "coalesce(sum(coalesce(length(distinct content), 0)), 0)"
charCount := "coalesce(sum(coalesce(charcount(distinct content), 0)), 0)"
wordCount := "coalesce(sum(wordcount(distinct content)), 0) as wordcount"
wordsPerPost := "coalesce(round(wordcount/postcount,0), 0)"
type statsTableType struct {

7
database.go
View File

@ -47,13 +47,18 @@ func (a *goBlog) openDatabase(file string, logging bool) (*database, error) {
dbDriverName := generateRandomString(15)
sql.Register("goblog_db_"+dbDriverName, &sqlite.SQLiteDriver{
ConnectHook: func(c *sqlite.SQLiteConn) error {
// Depends on app
if err := c.RegisterFunc("mdtext", a.renderText, true); err != nil {
return err
}
// Independent
if err := c.RegisterFunc("tolocal", toLocalSafe, true); err != nil {
return err
}
if err := c.RegisterFunc("wordcount", wordCount, true); err != nil {
return err
}
if err := c.RegisterFunc("mdtext", a.renderText, true); err != nil {
if err := c.RegisterFunc("charcount", charCount, true); err != nil {
return err
}
return nil

16
database_test.go
View File

@ -14,27 +14,27 @@ func Test_database(t *testing.T) {
db, err := app.openDatabase(":memory:", false)
if err != nil {
t.Errorf("Error: %v", err)
t.Fatalf("Error: %v", err)
}
_, err = db.execMulti("create table test(test text);")
if err != nil {
t.Errorf("Error: %v", err)
t.Fatalf("Error: %v", err)
}
_, err = db.exec("insert into test (test) values ('Test')")
if err != nil {
t.Errorf("Error: %v", err)
t.Fatalf("Error: %v", err)
}
row, err := db.queryRow("select count(test) from test")
if err != nil {
t.Errorf("Error: %v", err)
t.Fatalf("Error: %v", err)
}
var test1 int
err = row.Scan(&test1)
if err != nil {
t.Errorf("Error: %v", err)
t.Fatalf("Error: %v", err)
}
if test1 != 1 {
t.Error("Wrong result")
@ -42,7 +42,7 @@ func Test_database(t *testing.T) {
rows, err := db.query("select count(test), test from test")
if err != nil {
t.Errorf("Error: %v", err)
t.Fatalf("Error: %v", err)
}
var test2 int
var testStr string
@ -51,7 +51,7 @@ func Test_database(t *testing.T) {
}
err = rows.Scan(&test2, &testStr)
if err != nil {
t.Errorf("Error: %v", err)
t.Fatalf("Error: %v", err)
}
if test2 != 1 || testStr != "Test" {
t.Error("Wrong result")
@ -59,7 +59,7 @@ func Test_database(t *testing.T) {
err = db.close()
if err != nil {
t.Errorf("Error: %v", err)
t.Fatalf("Error: %v", err)
}
})
}

7
go.mod
View File

@ -40,7 +40,7 @@ require (
github.com/mitchellh/go-server-timing v1.0.1
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/paulmach/go.geojson v1.4.0
github.com/pelletier/go-toml v1.9.2 // indirect
github.com/pelletier/go-toml v1.9.3 // indirect
github.com/pquerna/otp v1.3.0
github.com/schollz/sqlite3dump v1.2.4
github.com/smartystreets/assertions v1.2.0 // indirect
@ -49,6 +49,7 @@ require (
github.com/spf13/cast v1.3.1
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.7.1
github.com/stretchr/testify v1.7.0
github.com/tdewolff/minify/v2 v2.9.17
github.com/thoas/go-funk v0.8.0
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
@ -58,9 +59,9 @@ require (
github.com/yuin/goldmark-emoji v1.0.2-0.20210607094911-0487583eca38
go.uber.org/atomic v1.8.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b
golang.org/x/net v0.0.0-20210614182718-04defd469f4e
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210611083646-a4fc73990273 // indirect
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/ini.v1 v1.62.0 // indirect

11
go.sum
View File

@ -259,8 +259,8 @@ github.com/paulmach/go.geojson v1.4.0 h1:5x5moCkCtDo5x8af62P9IOAYGQcYHtxz2QJ3x1D
github.com/paulmach/go.geojson v1.4.0/go.mod h1:YaKx1hKpWF+T2oj2lFJPsW/t1Q5e1jQI61eoQSTwpIs=
github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.9.2 h1:7NiByeVF4jKSG1lDF3X8LTIkq2/bu+1uYbIm1eS5tzk=
github.com/pelletier/go-toml v1.9.2/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@ -410,8 +410,9 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b h1:k+E048sYJHyVnsr1GDrRZWQ32D2C7lWs9JRc0bel53A=
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -444,8 +445,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210611083646-a4fc73990273 h1:faDu4veV+8pcThn4fewv6TVlNCezafGoC1gM/mxQLbQ=
golang.org/x/sys v0.0.0-20210611083646-a4fc73990273/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

8
markdown_test.go
View File

@ -22,7 +22,7 @@ func Test_markdown(t *testing.T) {
rendered, err := app.renderMarkdown("[Relative](/relative)", false)
if err != nil {
t.Errorf("Error: %v", err)
t.Fatalf("Error: %v", err)
}
if !strings.Contains(string(rendered), `href="/relative"`) {
t.Errorf("Wrong result, got %v", string(rendered))
@ -30,7 +30,7 @@ func Test_markdown(t *testing.T) {
rendered, err = app.renderMarkdown("[Relative](/relative)", true)
if err != nil {
t.Errorf("Error: %v", err)
t.Fatalf("Error: %v", err)
}
if !strings.Contains(string(rendered), `href="https://example.com/relative"`) {
t.Errorf("Wrong result, got %v", string(rendered))
@ -43,7 +43,7 @@ func Test_markdown(t *testing.T) {
rendered, err = app.renderMarkdown("[External](https://example.com)", true)
if err != nil {
t.Errorf("Error: %v", err)
t.Fatalf("Error: %v", err)
}
if !strings.Contains(string(rendered), `target="_blank"`) {
t.Errorf("Wrong result, got %v", string(rendered))
@ -53,7 +53,7 @@ func Test_markdown(t *testing.T) {
rendered, err = app.renderMarkdown(`[With title](https://example.com "Test-Title")`, true)
if err != nil {
t.Errorf("Error: %v", err)
t.Fatalf("Error: %v", err)
}
if !strings.Contains(string(rendered), `title="Test-Title"`) {
t.Errorf("Wrong result, got %v", string(rendered))

75
postsDb.go
View File

@ -11,6 +11,7 @@ import (
"time"
"github.com/araddon/dateparse"
"github.com/thoas/go-funk"
)
func (a *goBlog) checkPost(p *post) (err error) {
@ -124,10 +125,28 @@ type postCreationOptions struct {
var postCreationMutex sync.Mutex
func (a *goBlog) createOrReplacePost(p *post, o *postCreationOptions) error {
err := a.checkPost(p)
if err != nil {
// Check post
if err := a.checkPost(p); err != nil {
return err
}
// Save to db
if err := a.db.savePost(p, o); err != nil {
return err
}
// Trigger hooks
if p.Status == statusPublished {
if o.new || o.oldStatus == statusDraft {
defer a.postPostHooks(p)
} else {
defer a.postUpdateHooks(p)
}
}
// Reload router
return a.reloadRouter()
}
// Save check post to database
func (db *database) savePost(p *post, o *postCreationOptions) error {
// Prevent bad things
postCreationMutex.Lock()
defer postCreationMutex.Unlock()
@ -135,7 +154,7 @@ func (a *goBlog) createOrReplacePost(p *post, o *postCreationOptions) error {
if o.new || (p.Path != o.oldPath) {
// Post is new or post path was changed
newPathExists := false
row, err := a.db.queryRow("select exists(select 1 from posts where path = @path)", sql.Named("path", p.Path))
row, err := db.queryRow("select exists(select 1 from posts where path = @path)", sql.Named("path", p.Path))
if err != nil {
return err
}
@ -169,39 +188,39 @@ func (a *goBlog) createOrReplacePost(p *post, o *postCreationOptions) error {
}
}
// Execute
_, err = a.db.execMulti(sqlBuilder.String(), sqlArgs...)
if err != nil {
if _, err := db.execMulti(sqlBuilder.String(), sqlArgs...); err != nil {
return err
}
// Update FTS index, trigger hooks and reload router
a.db.rebuildFTSIndex()
if p.Status == statusPublished {
if o.new || o.oldStatus == statusDraft {
defer a.postPostHooks(p)
} else {
defer a.postUpdateHooks(p)
}
}
return a.reloadRouter()
// Update FTS index
db.rebuildFTSIndex()
return nil
}
func (a *goBlog) deletePost(path string) error {
if path == "" {
return nil
}
p, err := a.db.getPost(path)
if err != nil {
p, err := a.db.deletePost(path)
if err != nil || p == nil {
return err
}
_, err = a.db.exec("delete from posts where path = @path", sql.Named("path", p.Path))
if err != nil {
return err
}
a.db.rebuildFTSIndex()
defer a.postDeleteHooks(p)
return a.reloadRouter()
}
func (db *database) deletePost(path string) (*post, error) {
if path == "" {
return nil, nil
}
p, err := db.getPost(path)
if err != nil {
return nil, err
}
_, err = db.exec("delete from posts where path = @path", sql.Named("path", p.Path))
if err != nil {
return nil, err
}
db.rebuildFTSIndex()
return p, nil
}
type postsRequestConfig struct {
search string
blog string
@ -366,9 +385,9 @@ func (d *database) allPostPaths(status postStatus) ([]string, error) {
}
func (a *goBlog) getRandomPostPath(blog string) (string, error) {
var sections []string
for sectionKey := range a.cfg.Blogs[blog].Sections {
sections = append(sections, sectionKey)
sections, ok := funk.Keys(a.cfg.Blogs[blog].Sections).([]string)
if !ok {
return "", errors.New("no sections")
}
posts, err := a.db.getPosts(&postsRequestConfig{randomOrder: true, limit: 1, blog: blog, sections: sections})
if err != nil {

177
postsDb_test.go Normal file
View File

@ -0,0 +1,177 @@
package main
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_postsDb(t *testing.T) {
is := assert.New(t)
must := require.New(t)
app := &goBlog{
cfg: &config{
Blogs: map[string]*configBlog{
"en": {
Sections: map[string]*section{
"test": {},
},
},
},
},
}
app.setInMemoryDatabase()
now := toLocalSafe(time.Now().String())
nowPlus1Hour := toLocalSafe(time.Now().Add(1 * time.Hour).String())
// Save post
err := app.db.savePost(&post{
Path: "/test/abc",
Content: "ABC",
Published: now,
Updated: nowPlus1Hour,
Blog: "en",
Section: "test",
Status: statusDraft,
Parameters: map[string][]string{
"title": {"Title"},
},
}, &postCreationOptions{new: true})
must.NoError(err)
// Check post
p, err := app.db.getPost("/test/abc")
is.NoError(err)
is.Equal("/test/abc", p.Path)
is.Equal("ABC", p.Content)
is.Equal(now, p.Published)
is.Equal(nowPlus1Hour, p.Updated)
is.Equal("en", p.Blog)
is.Equal("test", p.Section)
is.Equal(statusDraft, p.Status)
is.Equal("Title", p.title())
// Check number of post paths
pp, err := app.db.allPostPaths(statusDraft)
is.NoError(err)
if is.Len(pp, 1) {
is.Equal("/test/abc", pp[0])
}
pp, err = app.db.allPostPaths(statusPublished)
is.NoError(err)
is.Len(pp, 0)
// Check drafts
drafts := app.db.getDrafts("en")
is.Len(drafts, 1)
// Delete post
_, err = app.db.deletePost("/test/abc")
must.NoError(err)
// Check that there is no post
count, err := app.db.countPosts(&postsRequestConfig{})
is.NoError(err)
is.Equal(0, count)
// Save published post
err = app.db.savePost(&post{
Path: "/test/abc",
Content: "ABC",
Published: "2021-06-10 10:00:00",
Updated: "2021-06-15 10:00:00",
Blog: "en",
Section: "test",
Status: statusPublished,
Parameters: map[string][]string{
"tags": {"Test", "Blog"},
},
}, &postCreationOptions{new: true})
must.NoError(err)
// Check that there is a new post
count, err = app.db.countPosts(&postsRequestConfig{})
if is.NoError(err) {
is.Equal(1, count)
}
// Check random post path
rp, err := app.getRandomPostPath("en")
if is.NoError(err) {
is.Equal("/test/abc", rp)
}
// Check taxonomies
tags, err := app.db.allTaxonomyValues("en", "tags")
if is.NoError(err) {
is.Len(tags, 2)
is.Equal([]string{"Test", "Blog"}, tags)
}
// Check based on date
count, err = app.db.countPosts(&postsRequestConfig{
publishedYear: 2020,
})
if is.NoError(err) {
is.Equal(0, count)
}
count, err = app.db.countPosts(&postsRequestConfig{
publishedYear: 2021,
})
if is.NoError(err) {
is.Equal(1, count)
}
// Check dates
dates, err := app.db.allPublishedDates("en")
if is.NoError(err) && is.NotEmpty(dates) {
is.Equal(publishedDate{year: 2021, month: 6, day: 10}, dates[0])
}
// Check based on tags
count, err = app.db.countPosts(&postsRequestConfig{
parameter: "tags",
parameterValue: "ABC",
})
if is.NoError(err) {
is.Equal(0, count)
}
count, err = app.db.countPosts(&postsRequestConfig{
parameter: "tags",
parameterValue: "Blog",
})
if is.NoError(err) {
is.Equal(1, count)
}
}
func Test_ftsWithoutTitle(t *testing.T) {
// Added because there was a bug where there were no search results without title
app := &goBlog{}
app.setInMemoryDatabase()
err := app.db.savePost(&post{
Path: "/test/abc",
Content: "ABC",
Published: toLocalSafe(time.Now().String()),
Updated: toLocalSafe(time.Now().Add(1 * time.Hour).String()),
Blog: "en",
Section: "test",
Status: statusDraft,
}, &postCreationOptions{new: true})
require.NoError(t, err)
ps, err := app.db.getPosts(&postsRequestConfig{
search: "ABC",
})
assert.NoError(t, err)
assert.Len(t, ps, 1)
}

35
robotstxt_test.go
View File

@ -1,26 +1,18 @@
package main
import (
"io"
"net/http"
"net/http/httptest"
"reflect"
"testing"
"github.com/stretchr/testify/assert"
)
func Test_robotsTXT(t *testing.T) {
testRecorder := httptest.NewRecorder()
testRequest := httptest.NewRequest(http.MethodGet, "/robots.txt", nil)
servePrivateRobotsTXT(testRecorder, testRequest)
testResult := testRecorder.Result()
if sc := testResult.StatusCode; sc != 200 {
t.Errorf("Wrong status code, got: %v", sc)
}
if rb, _ := io.ReadAll(testResult.Body); !reflect.DeepEqual(rb, []byte("User-agent: *\nDisallow: /")) {
t.Errorf("Wrong response body, got: %v", rb)
}
h := http.HandlerFunc(servePrivateRobotsTXT)
assert.HTTPStatusCode(t, h, http.MethodGet, "", nil, 200)
txt := assert.HTTPBody(h, http.MethodGet, "", nil)
assert.Equal(t, "User-agent: *\nDisallow: /", txt)
app := &goBlog{
cfg: &config{
@ -30,16 +22,9 @@ func Test_robotsTXT(t *testing.T) {
},
}
testRecorder = httptest.NewRecorder()
testRequest = httptest.NewRequest(http.MethodGet, "/robots.txt", nil)
h = http.HandlerFunc(app.serveRobotsTXT)
assert.HTTPStatusCode(t, h, http.MethodGet, "", nil, 200)
txt = assert.HTTPBody(h, http.MethodGet, "", nil)
assert.Equal(t, "User-agent: *\nSitemap: https://example.com/sitemap.xml", txt)
app.serveRobotsTXT(testRecorder, testRequest)
testResult = testRecorder.Result()
if sc := testResult.StatusCode; sc != 200 {
t.Errorf("Wrong status code, got: %v", sc)
}
if rb, _ := io.ReadAll(testResult.Body); !reflect.DeepEqual(rb, []byte("User-agent: *\nSitemap: https://example.com/sitemap.xml")) {
t.Errorf("Wrong response body, got: %v", string(rb))
}
}

2
telegram_test.go
View File

@ -82,7 +82,7 @@ func Test_configTelegram_send(t *testing.T) {
err := tg.send("Message", "HTML")
if err != nil {
t.Errorf("Error: %v", err)
t.Fatalf("Error: %v", err)
}
if fakeAppHttpClient.req == nil {

11
utils.go
View File

@ -6,6 +6,7 @@ import (
"net/url"
"sort"
"strings"
"unicode"
"github.com/PuerkitoBio/goquery"
"github.com/araddon/dateparse"
@ -162,3 +163,13 @@ type stringPair struct {
func wordCount(s string) int {
return len(strings.Fields(s))
}
// Count all letters and numbers in string
func charCount(s string) (count int) {
for _, r := range s {
if unicode.IsLetter(r) || unicode.IsNumber(r) {
count++
}
}
return count
}

10
utils_test.go
View File

@ -5,6 +5,8 @@ import (
"net/http/httptest"
"reflect"
"testing"
"github.com/stretchr/testify/assert"
)
func Test_urlize(t *testing.T) {
@ -63,9 +65,11 @@ func Test_isAbsoluteURL(t *testing.T) {
}
func Test_wordCount(t *testing.T) {
if wordCount("abc def abc") != 3 {
t.Error("Wrong result")
}
assert.Equal(t, 3, wordCount("abc def abc"))
}
func Test_charCount(t *testing.T) {
assert.Equal(t, 4, charCount(" t e\n s t €.☺️"))
}
func Test_allLinksFromHTMLString(t *testing.T) {