Various small improvements and fixes

This commit is contained in:
Jan-Lukas Else 2021-12-30 12:40:21 +01:00
parent e994e5400d
commit 7fe07014f5
14 changed files with 142 additions and 115 deletions

View File

@ -50,13 +50,13 @@ func (a *goBlog) serveBlogrollExport(w http.ResponseWriter, r *http.Request) {
return
}
w.Header().Set(contentType, contenttype.XMLUTF8)
var opmlBytes bytes.Buffer
_ = opml.Render(&opmlBytes, &opml.OPML{
var opmlBuffer bytes.Buffer
_ = opml.Render(&opmlBuffer, &opml.OPML{
Version: "2.0",
DateCreated: time.Now().UTC(),
Outlines: outlines.([]*opml.Outline),
})
_, _ = a.min.Write(w, contenttype.XML, opmlBytes.Bytes())
_ = a.min.Minify(contenttype.XML, w, &opmlBuffer)
}
func (a *goBlog) getBlogrollOutlines(blog string) ([]*opml.Outline, error) {

View File

@ -3,7 +3,6 @@ package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/binary"
"fmt"
"io"
@ -57,11 +56,7 @@ func (a *goBlog) cacheMiddleware(next http.Handler) http.Handler {
return
}
// Do checks
if !(r.Method == http.MethodGet || r.Method == http.MethodHead) {
next.ServeHTTP(w, r)
return
}
if r.URL.Query().Get("cache") == "0" || r.URL.Query().Get("cache") == "false" {
if !cacheable(r) {
next.ServeHTTP(w, r)
return
}
@ -71,6 +66,7 @@ func (a *goBlog) cacheMiddleware(next http.Handler) http.Handler {
setLoggedIn(r, false)
} else {
if a.isLoggedIn(r) {
// Don't cache logged in requests
next.ServeHTTP(w, r)
return
}
@ -82,11 +78,8 @@ func (a *goBlog) cacheMiddleware(next http.Handler) http.Handler {
return a.cache.getCache(key, next, r), nil
})
ci := cacheInterface.(*cacheItem)
// copy cached headers
for k, v := range ci.header {
w.Header()[k] = v
}
a.cache.setCacheHeaders(w, ci)
// copy and set headers
a.cache.setHeaders(w, ci)
// check conditional request
if ifNoneMatchHeader := r.Header.Get("If-None-Match"); ifNoneMatchHeader != "" && ifNoneMatchHeader == ci.eTag {
// send 304
@ -107,6 +100,16 @@ func (a *goBlog) cacheMiddleware(next http.Handler) http.Handler {
})
}
func cacheable(r *http.Request) bool {
if !(r.Method == http.MethodGet || r.Method == http.MethodHead) {
return false
}
if r.URL.Query().Get("cache") == "0" || r.URL.Query().Get("cache") == "false" {
return false
}
return true
}
func cacheKey(r *http.Request) string {
var buf strings.Builder
// Special cases
@ -126,7 +129,12 @@ func cacheKey(r *http.Request) string {
return buf.String()
}
func (c *cache) setCacheHeaders(w http.ResponseWriter, cache *cacheItem) {
func (c *cache) setHeaders(w http.ResponseWriter, cache *cacheItem) {
// Copy headers
for k, v := range cache.header.Clone() {
w.Header()[k] = v
}
// Set cache headers
w.Header().Set("ETag", cache.eTag)
w.Header().Set("Last-Modified", cache.creationTime.UTC().Format(http.TimeFormat))
if w.Header().Get("Cache-Control") == "" {
@ -163,7 +171,7 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *
if item == nil {
// No cache available
// Make and use copy of r
cr := r.Clone(r.Context())
cr := r.Clone(valueOnlyContext{r.Context()})
// Remove problematic headers
cr.Header.Del("If-Modified-Since")
cr.Header.Del("If-Unmodified-Since")
@ -180,9 +188,7 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *
_ = result.Body.Close()
eTag := result.Header.Get("ETag")
if eTag == "" {
h := sha256.New()
_, _ = io.Copy(h, bytes.NewReader(body))
eTag = fmt.Sprintf("%x", h.Sum(nil))
eTag, _ = getSHA256(bytes.NewReader(body))
}
lastMod := time.Now()
if lm := result.Header.Get("Last-Modified"); lm != "" {
@ -190,12 +196,12 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *
lastMod = parsedTime
}
}
exp, _ := cr.Context().Value(cacheExpirationKey).(int)
// Remove problematic headers
result.Header.Del("Accept-Ranges")
result.Header.Del("ETag")
result.Header.Del("Last-Modified")
// Create cache item
exp, _ := cr.Context().Value(cacheExpirationKey).(int)
item = &cacheItem{
expiration: exp,
creationTime: lastMod,

View File

@ -1,8 +1,6 @@
package main
import (
"context"
"net/http"
"path/filepath"
"testing"
)
@ -12,7 +10,3 @@ func createDefaultTestConfig(t *testing.T) *config {
c.Db.File = filepath.Join(t.TempDir(), "blog.db")
return c
}
func reqWithDefaultBlog(req *http.Request) *http.Request {
return req.WithContext(context.WithValue(req.Context(), blogKey, "default"))
}

View File

@ -150,17 +150,19 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) {
a.serveError(w, r, err.Error(), http.StatusBadRequest)
return
}
gpx, err := io.ReadAll(file)
originalGpx, err := io.ReadAll(file)
if err != nil {
a.serveError(w, r, err.Error(), http.StatusBadRequest)
return
}
var gpxBuffer bytes.Buffer
_, _ = a.min.Write(&gpxBuffer, contenttype.XML, gpx)
resultMap := map[string]string{
"gpx": gpxBuffer.String(),
minifiedGpx, err := a.min.MinifyString(contenttype.XML, string(originalGpx))
if err != nil {
a.serveError(w, r, err.Error(), http.StatusInternalServerError)
return
}
resultBytes, err := yaml.Marshal(resultMap)
resultBytes, err := yaml.Marshal(map[string]string{
"gpx": minifiedGpx,
})
if err != nil {
a.serveError(w, r, err.Error(), http.StatusBadRequest)
return

View File

@ -1,6 +1,7 @@
package main
import (
"bytes"
"net/http"
"strings"
"time"
@ -49,25 +50,26 @@ func (a *goBlog) generateFeed(blog string, f feedType, w http.ResponseWriter, r
})
}
var err error
var feedString, feedMediaType string
var feedBuffer bytes.Buffer
var feedMediaType string
switch f {
case rssFeed:
feedMediaType = contenttype.RSS
feedString, err = feed.ToRss()
err = feed.WriteRss(&feedBuffer)
case atomFeed:
feedMediaType = contenttype.ATOM
feedString, err = feed.ToAtom()
err = feed.WriteAtom(&feedBuffer)
case jsonFeed:
feedMediaType = contenttype.JSONFeed
feedString, err = feed.ToJSON()
err = feed.WriteJSON(&feedBuffer)
default:
a.serve404(w, r)
return
}
if err != nil {
w.Header().Del(contentType)
a.serveError(w, r, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set(contentType, feedMediaType+contenttype.CharsetUtf8Suffix)
_, _ = a.min.Write(w, feedMediaType, []byte(feedString))
_ = a.min.Minify(feedMediaType, w, &feedBuffer)
}

4
go.mod
View File

@ -33,7 +33,7 @@ require (
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
github.com/lopezator/migrator v0.3.0
github.com/mattn/go-sqlite3 v1.14.9
github.com/mattn/go-sqlite3 v1.14.10
github.com/microcosm-cc/bluemonday v1.0.16
github.com/mmcdole/gofeed v1.1.3
github.com/paulmach/go.geojson v1.4.0
@ -44,7 +44,7 @@ require (
github.com/spf13/cast v1.4.1
github.com/spf13/viper v1.10.1
github.com/stretchr/testify v1.7.0
github.com/tdewolff/minify/v2 v2.9.24
github.com/tdewolff/minify/v2 v2.9.26
github.com/thoas/go-funk v0.9.1
github.com/tkrajina/gpxgo v1.1.2
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80

8
go.sum
View File

@ -303,8 +303,8 @@ github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk=
github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y=
github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43 h1:WgyLFv10Ov49JAQI/ZLUkCZ7VJS3r74hwFIGXJsgZlY=
github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo=
@ -400,8 +400,8 @@ github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85 h1:zrsUcqrG2uQ
github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0=
github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
github.com/tdewolff/minify/v2 v2.9.24 h1:4wlbX+U5IgVa8fH//mlwzv+2g47MN7lu3s9HClAHc28=
github.com/tdewolff/minify/v2 v2.9.24/go.mod h1:L/bwPtsU/Xx30MxCndlClCMMiLbqROgkR4vZT+QIGXA=
github.com/tdewolff/minify/v2 v2.9.26 h1:eQy5DRs7vH5pxkF0xtFv29bOgyBEL2vInj4v1bFkK88=
github.com/tdewolff/minify/v2 v2.9.26/go.mod h1:L/bwPtsU/Xx30MxCndlClCMMiLbqROgkR4vZT+QIGXA=
github.com/tdewolff/parse/v2 v2.5.26 h1:a/q3lwDCi4GIQ+sSbs4UOHuObhqp8GHAhfqop/zDyQQ=
github.com/tdewolff/parse/v2 v2.5.26/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho=
github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4=

View File

@ -1,6 +1,7 @@
package main
import (
"bytes"
"fmt"
"net/http"
@ -11,14 +12,15 @@ func (a *goBlog) serveOpenSearch(w http.ResponseWriter, r *http.Request) {
_, b := a.getBlog(r)
title := a.renderMdTitle(b.Title)
sURL := a.getFullAddress(b.getRelativePath(defaultIfEmpty(b.Search.Path, defaultSearchPath)))
xml := fmt.Sprintf("<?xml version=\"1.0\"?><OpenSearchDescription xmlns=\"http://a9.com/-/spec/opensearch/1.1/\" xmlns:moz=\"http://www.mozilla.org/2006/browser/search/\">"+
var buf bytes.Buffer
_, _ = fmt.Fprintf(&buf, "<?xml version=\"1.0\"?><OpenSearchDescription xmlns=\"http://a9.com/-/spec/opensearch/1.1/\" xmlns:moz=\"http://www.mozilla.org/2006/browser/search/\">"+
"<ShortName>%s</ShortName><Description>%s</Description>"+
"<Url type=\"text/html\" method=\"post\" template=\"%s\"><Param name=\"q\" value=\"{searchTerms}\" /></Url>"+
"<moz:SearchForm>%s</moz:SearchForm>"+
"</OpenSearchDescription>",
title, title, sURL, sURL)
w.Header().Set(contentType, "application/opensearchdescription+xml")
_, _ = a.min.Write(w, contenttype.XML, []byte(xml))
_ = a.min.Minify(contenttype.XML, w, &buf)
}
func openSearchUrl(b *configBlog) string {

View File

@ -50,3 +50,7 @@ func (m *Minifier) MinifyBytes(mediatype string, b []byte) ([]byte, error) {
func (m *Minifier) MinifyString(mediatype string, s string) (string, error) {
return m.Get().String(mediatype, s)
}
func (m *Minifier) Minify(mediatype string, w io.Writer, r io.Reader) error {
return m.Get().Minify(mediatype, w, r)
}

View File

@ -135,16 +135,14 @@ func (a *goBlog) renderWithStatusCode(w http.ResponseWriter, r *http.Request, st
a.checkRenderData(r, data)
// Set content type
w.Header().Set(contentType, contenttype.HTMLUTF8)
// Minify and write response
var tw bytes.Buffer
err := a.templates[template].ExecuteTemplate(&tw, template, data)
if err != nil {
// Render template and write minified HTML
var templateBuffer bytes.Buffer
if err := a.templates[template].ExecuteTemplate(&templateBuffer, template, data); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(statusCode)
_, err = a.min.Write(w, contenttype.HTML, tw.Bytes())
if err != nil {
if err := a.min.Minify(contenttype.HTML, w, &templateBuffer); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}

View File

@ -186,7 +186,7 @@ func (a *goBlog) writeSitemapXML(w http.ResponseWriter, sm interface{}) {
buf.WriteString(a.assetFileName("sitemap.xsl"))
buf.WriteString(`" ?>`)
_ = xml.NewEncoder(&buf).Encode(sm)
_, _ = a.min.Write(w, contenttype.XML, buf.Bytes())
_ = a.min.Minify(contenttype.XML, w, &buf)
}
const sitemapDatePathsSql = `

View File

@ -1,8 +1,8 @@
package main
import (
"crypto/sha1"
"fmt"
"bytes"
"io"
"mime"
"net/http"
"os"
@ -26,10 +26,18 @@ func (a *goBlog) initTemplateAssets() (err error) {
a.assetFiles = map[string]*assetFile{}
err = filepath.Walk(assetsFolder, func(path string, info os.FileInfo, err error) error {
if info.Mode().IsRegular() {
compiled, err := a.compileAsset(path)
// Open file
file, err := os.Open(path)
if err != nil {
return err
}
// Compile asset and close file
compiled, err := a.compileAsset(path, file)
_ = file.Close()
if err != nil {
return err
}
// Add to map
if compiled != "" {
a.assetFileNames[strings.TrimPrefix(path, assetsFolder+"/")] = compiled
}
@ -47,43 +55,39 @@ func (a *goBlog) initTemplateAssets() (err error) {
return nil
}
func (a *goBlog) compileAsset(name string) (string, error) {
content, err := os.ReadFile(name)
if err != nil {
return "", err
}
func (a *goBlog) compileAsset(name string, read io.Reader) (string, error) {
ext := path.Ext(name)
compiledExt := ext
var contentBuffer bytes.Buffer
switch ext {
case ".js":
content, err = a.min.MinifyBytes(contenttype.JS, content)
if err != nil {
if err := a.min.Minify(contenttype.JS, &contentBuffer, read); err != nil {
return "", err
}
case ".css":
content, err = a.min.MinifyBytes(contenttype.CSS, content)
if err != nil {
if err := a.min.Minify(contenttype.CSS, &contentBuffer, read); err != nil {
return "", err
}
case ".xml", ".xsl":
content, err = a.min.MinifyBytes(contenttype.XML, content)
if err != nil {
if err := a.min.Minify(contenttype.XML, &contentBuffer, read); err != nil {
return "", err
}
default:
// Do nothing
if _, err := io.Copy(&contentBuffer, read); err != nil {
return "", err
}
}
// Hashes
sha1Hash := sha1.New()
if _, err := sha1Hash.Write(content); err != nil {
hash, err := getSHA256(bytes.NewReader(contentBuffer.Bytes()))
if err != nil {
return "", err
}
// File name
compiledFileName := fmt.Sprintf("%x", sha1Hash.Sum(nil)) + compiledExt
compiledFileName := hash + compiledExt
// Create struct
a.assetFiles[compiledFileName] = &assetFile{
contentType: mime.TypeByExtension(compiledExt),
body: content,
body: contentBuffer.Bytes(),
}
return compiledFileName, err
}
@ -114,8 +118,9 @@ func (a *goBlog) serveAsset(w http.ResponseWriter, r *http.Request) {
}
func (a *goBlog) initChromaCSS() error {
chromaPath := "css/chroma.css"
// Check if file already exists
if _, ok := a.assetFiles["css/chroma.css"]; ok {
if _, ok := a.assetFiles[chromaPath]; ok {
return nil
}
// Initialize the style
@ -124,25 +129,17 @@ func (a *goBlog) initChromaCSS() error {
if err != nil {
return err
}
// Create a temporary file
chromaTempFile, err := os.CreateTemp("", "chroma-*.css")
if err != nil {
// Write the CSS to a buffer
var cssBuffer bytes.Buffer
if err = chromahtml.New(chromahtml.ClassPrefix("c-")).WriteCSS(&cssBuffer, chromaStyle); err != nil {
return err
}
chromaTempFileName := chromaTempFile.Name()
// Write the CSS to the file
err = chromahtml.New(chromahtml.ClassPrefix("c-")).WriteCSS(chromaTempFile, chromaStyle)
if err != nil {
return err
}
// Close the file
_ = chromaTempFile.Close()
// Compile asset
compiled, err := a.compileAsset(chromaTempFileName)
_ = os.Remove(chromaTempFileName)
compiled, err := a.compileAsset(chromaPath, &cssBuffer)
if err != nil {
return err
}
a.assetFileNames["css/chroma.css"] = compiled
// Add to map
a.assetFileNames[chromaPath] = compiled
return nil
}

58
tts.go
View File

@ -1,14 +1,16 @@
package main
import (
"bytes"
"context"
"encoding/base64"
"errors"
"html"
"io"
"log"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"github.com/carlmjohnson/requests"
)
@ -54,32 +56,29 @@ func (a *goBlog) createPostTTSAudio(p *post) error {
if lang == "" {
lang = "en"
}
text := a.renderMdTitle(p.Title()) + "\n\n" + cleanHTMLText(string(a.postHtml(p, false)))
// Generate audio file
tmpDir, err := os.MkdirTemp("", "")
if err != nil {
return err
}
defer func() {
_ = os.RemoveAll(tmpDir)
}()
outputFileName := filepath.Join(tmpDir, "audio.mp3")
err = a.createTTSAudio(lang, text, outputFileName)
// Build SSML
var ssml strings.Builder
ssml.WriteString("<speak>")
ssml.WriteString(html.EscapeString(a.renderMdTitle(p.Title())))
ssml.WriteString("<break time=\"1s\"/>")
ssml.WriteString(html.EscapeString(cleanHTMLText(string(a.postHtml(p, false)))))
ssml.WriteString("</speak>")
// Generate audio
var audioBuffer bytes.Buffer
err := a.createTTSAudio(lang, ssml.String(), &audioBuffer)
if err != nil {
return err
}
// Save new audio file
file, err := os.Open(outputFileName)
// Save audio
audioReader := bytes.NewReader(audioBuffer.Bytes())
fileHash, err := getSHA256(audioReader)
if err != nil {
return err
}
fileHash, err := getSHA256(file)
if err != nil {
return err
}
loc, err := a.saveMediaFile(fileHash+".mp3", file)
loc, err := a.saveMediaFile(fileHash+".mp3", audioReader)
if err != nil {
return err
}
@ -133,7 +132,7 @@ func (a *goBlog) deletePostTTSAudio(p *post) bool {
return true
}
func (a *goBlog) createTTSAudio(lang, text, outputFile string) error {
func (a *goBlog) createTTSAudio(lang, ssml string, w io.Writer) error {
// Check if Google Cloud TTS is enabled
gctts := a.cfg.TTS
if !gctts.Enabled || gctts.GoogleAPIKey == "" {
@ -144,20 +143,26 @@ func (a *goBlog) createTTSAudio(lang, text, outputFile string) error {
if lang == "" {
return errors.New("language not provided")
}
if text == "" {
if ssml == "" {
return errors.New("empty text")
}
if outputFile == "" {
return errors.New("output file not provided")
if w == nil {
return errors.New("writer not provided")
}
// Check max length
// TODO: Support longer texts by splitting into multiple requests
// if len(ssml) > 5000 {
// return errors.New("text is too long")
// }
// Create request body
body := map[string]interface{}{
"audioConfig": map[string]interface{}{
"audioEncoding": "MP3",
},
"input": map[string]interface{}{
"text": text,
"ssml": ssml,
},
"voice": map[string]interface{}{
"languageCode": lang,
@ -183,7 +188,8 @@ func (a *goBlog) createTTSAudio(lang, text, outputFile string) error {
if encoded, ok := response["audioContent"]; ok {
if encodedStr, ok := encoded.(string); ok {
if audio, err := base64.StdEncoding.DecodeString(encodedStr); err == nil {
return os.WriteFile(outputFile, audio, os.ModePerm)
_, err := w.Write(audio)
return err
} else {
return err
}

View File

@ -1,6 +1,7 @@
package main
import (
"context"
"crypto/sha256"
"errors"
"fmt"
@ -263,8 +264,7 @@ func htmlText(s string) string {
}
func cleanHTMLText(s string) string {
s = bluemonday.UGCPolicy().Sanitize(s)
return htmlText(s)
return htmlText(bluemonday.UGCPolicy().Sanitize(s))
}
func defaultIfEmpty(s, d string) string {
@ -327,3 +327,19 @@ func saveToFile(reader io.Reader, fileName string) error {
_, err = io.Copy(out, reader)
return err
}
type valueOnlyContext struct {
context.Context
}
func (valueOnlyContext) Deadline() (deadline time.Time, ok bool) {
return
}
func (valueOnlyContext) Done() <-chan struct{} {
return nil
}
func (valueOnlyContext) Err() error {
return nil
}