jlelse
/
GoBlog
mirror of https://github.com/jlelse/GoBlog
1
Fork 0
Code Packages Releases Activity

More io streaming

This commit is contained in:
Jan-Lukas Else 2022-02-11 19:39:07 +01:00
parent 3d5cb1ab4a
commit 6e767e3612
13 changed files with 115 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cache.go
View File

@ -1,8 +1,8 @@
package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/binary"
"fmt"
"io"
@ -181,12 +181,9 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *
next.ServeHTTP(recorder, cr)
// Cache values from recorder
result := recorder.Result()
body, _ := io.ReadAll(result.Body)
eTag := sha256.New()
body, _ := io.ReadAll(io.TeeReader(result.Body, eTag))
_ = result.Body.Close()
eTag := result.Header.Get("ETag")
if eTag == "" {
eTag, _ = getSHA256(bytes.NewReader(body))
}
lastMod := time.Now()
if lm := result.Header.Get(lastModified); lm != "" {
if parsedTime, te := dateparse.ParseLocal(lm); te == nil {
@ -202,7 +199,7 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *
item = &cacheItem{
expiration: exp,
creationTime: lastMod,
eTag: eTag,
eTag: fmt.Sprintf("%x", eTag.Sum(nil)),
code: result.StatusCode,
header: result.Header,
body: body,

46
editor.go
View File

@ -1,7 +1,6 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
@ -45,37 +44,34 @@ func (a *goBlog) serveEditorPreview(w http.ResponseWriter, r *http.Request) {
continue
}
// Create preview
preview, err := a.createMarkdownPreview(blog, string(message))
w, err := c.Writer(ctx, ws.MessageText)
if err != nil {
preview = []byte(err.Error())
break
}
// Write preview to socket
err = c.Write(ctx, ws.MessageText, preview)
a.createMarkdownPreview(w, blog, string(message))
err = w.Close()
if err != nil {
break
}
}
}
func (a *goBlog) createMarkdownPreview(blog string, markdown string) (rendered []byte, err error) {
func (a *goBlog) createMarkdownPreview(w io.Writer, blog string, markdown string) {
p := &post{
Blog: blog,
Content: markdown,
}
err = a.computeExtraPostParameters(p)
err := a.computeExtraPostParameters(p)
if err != nil {
return nil, err
_, _ = io.WriteString(w, err.Error())
return
}
if t := p.Title(); t != "" {
p.RenderedTitle = a.renderMdTitle(t)
}
// Render post
buf := bufferpool.Get()
hb := newHtmlBuilder(buf)
hb := newHtmlBuilder(w)
a.renderEditorPreview(hb, a.cfg.Blogs[blog], p)
rendered = buf.Bytes()
bufferpool.Put(buf)
return
}
func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) {
@ -94,7 +90,9 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) {
},
})
case "updatepost":
jsonBytes, err := json.Marshal(map[string]interface{}{
jsonBuf := bufferpool.Get()
defer bufferpool.Put(jsonBuf)
err := json.NewEncoder(jsonBuf).Encode(map[string]interface{}{
"action": actionUpdate,
"url": r.FormValue("url"),
"replace": map[string][]string{
@ -107,7 +105,7 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) {
a.serveError(w, r, err.Error(), http.StatusInternalServerError)
return
}
req, err := http.NewRequest(http.MethodPost, "", bytes.NewReader(jsonBytes))
req, err := http.NewRequest(http.MethodPost, "", jsonBuf)
if err != nil {
a.serveError(w, r, err.Error(), http.StatusInternalServerError)
return
@ -138,25 +136,15 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) {
a.serveError(w, r, err.Error(), http.StatusBadRequest)
return
}
originalGpx, err := io.ReadAll(file)
if err != nil {
a.serveError(w, r, err.Error(), http.StatusBadRequest)
return
}
minifiedGpx, err := a.min.MinifyString(contenttype.XML, string(originalGpx))
if err != nil {
a.serveError(w, r, err.Error(), http.StatusInternalServerError)
return
}
resultBytes, err := yaml.Marshal(map[string]string{
"gpx": minifiedGpx,
})
gpx, err := io.ReadAll(a.min.Reader(contenttype.XML, file))
if err != nil {
a.serveError(w, r, err.Error(), http.StatusBadRequest)
return
}
w.Header().Set(contentType, contenttype.TextUTF8)
_, _ = w.Write(resultBytes)
_ = yaml.NewEncoder(w).Encode(map[string]string{
"gpx": string(gpx),
})
default:
a.serveError(w, r, "Unknown editoraction", http.StatusBadRequest)
}

20
httpFs.go
View File

@ -1,7 +1,8 @@
package main
import (
"embed"
"io"
"io/fs"
"net/http"
"path"
"strings"
@ -9,24 +10,23 @@ import (
"go.goblog.app/app/pkgs/contenttype"
)
func (a *goBlog) serveFs(fs embed.FS, basePath string) http.HandlerFunc {
func (a *goBlog) serveFs(f fs.FS, basePath string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
fileName := strings.TrimPrefix(r.URL.Path, basePath)
fb, err := fs.ReadFile(fileName)
file, err := f.Open(fileName)
if err != nil {
a.serve404(w, r)
return
}
var read io.Reader = file
switch path.Ext(fileName) {
case ".js":
w.Header().Set(contentType, contenttype.JS)
_, _ = a.min.Write(w, contenttype.JSUTF8, fb)
w.Header().Set(contentType, contenttype.JSUTF8)
read = a.min.Reader(contenttype.JS, read)
case ".css":
w.Header().Set(contentType, contenttype.CSS)
_, _ = a.min.Write(w, contenttype.CSSUTF8, fb)
default:
w.Header().Set(contentType, http.DetectContentType(fb))
_, _ = w.Write(fb)
w.Header().Set(contentType, contenttype.CSSUTF8)
read = a.min.Reader(contenttype.CSS, read)
}
_, _ = io.Copy(w, read)
}
}

21
mediaCompression.go
View File

@ -3,12 +3,15 @@ package main
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
"log"
"net/http"
"github.com/carlmjohnson/requests"
"go.goblog.app/app/pkgs/bufferpool"
)
const defaultCompressionWidth = 2000
@ -167,14 +170,12 @@ func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http
return uploadCompressedFile(fileExtension, &imgBuffer, upload)
}
func uploadCompressedFile(fileExtension string, imgBuffer *bytes.Buffer, upload mediaStorageSaveFunc) (string, error) {
// Create reader from buffer
imgReader := bytes.NewReader(imgBuffer.Bytes())
// Get hash of compressed file
fileName, err := getSHA256(imgReader)
if err != nil {
return "", err
}
// Upload compressed file
return upload(fileName+"."+fileExtension, imgReader)
func uploadCompressedFile(fileExtension string, r io.Reader, upload mediaStorageSaveFunc) (string, error) {
// Copy file to temporary buffer to generate hash and filename
hash := sha256.New()
tempBuffer := bufferpool.Get()
defer bufferpool.Put(tempBuffer)
_, _ = io.Copy(io.MultiWriter(tempBuffer, hash), r)
// Upload buffer
return upload(fmt.Sprintf("%x.%s", hash.Sum(nil), fileExtension), tempBuffer)
}

20
mediaCompression_test.go
View File

@ -1,11 +1,11 @@
package main
import (
"crypto/sha256"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
@ -14,13 +14,9 @@ import (
func Test_compress(t *testing.T) {
fakeFileContent := "Test"
fakeFileName := filepath.Join(t.TempDir(), "test.jpg")
err := os.WriteFile(fakeFileName, []byte(fakeFileContent), 0666)
require.Nil(t, err)
fakeFile, err := os.Open(fakeFileName)
require.Nil(t, err)
fakeSha256, err := getSHA256(fakeFile)
require.Nil(t, err)
hash := sha256.New()
io.WriteString(hash, fakeFileContent)
fakeSha256 := fmt.Sprintf("%x", hash.Sum(nil))
var uf mediaStorageSaveFunc = func(filename string, f io.Reader) (location string, err error) {
return "https://example.com/" + filename, nil
@ -32,7 +28,7 @@ func Test_compress(t *testing.T) {
assert.Equal(t, "https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=2000,h=3000/https://example.com/original.jpg", r.URL.String())
rw.WriteHeader(http.StatusOK)
_, _ = rw.Write([]byte(fakeFileContent))
_, _ = io.WriteString(rw, fakeFileContent)
}))
cf := &cloudflare{}
@ -51,7 +47,7 @@ func Test_compress(t *testing.T) {
defer r.Body.Close()
var requestJson map[string]interface{}
err = json.Unmarshal(requestBody, &requestJson)
err := json.Unmarshal(requestBody, &requestJson)
require.Nil(t, err)
require.NotNil(t, requestJson)
@ -59,7 +55,7 @@ func Test_compress(t *testing.T) {
assert.Equal(t, "https://example.com/original.jpg", requestJson["url"])
rw.WriteHeader(http.StatusOK)
_, _ = rw.Write([]byte(fakeFileContent))
_, _ = io.WriteString(rw, fakeFileContent)
}))
cf := &shortpixel{"testkey"}

18
micropub.go
View File

@ -28,11 +28,11 @@ func (a *goBlog) serveMicropubQuery(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("q") {
case "config":
w.Header().Set(contentType, contenttype.JSONUTF8)
w.WriteHeader(http.StatusOK)
b, _ := json.Marshal(&micropubConfig{
mw := a.min.Writer(contenttype.JSON, w)
defer mw.Close()
_ = json.NewEncoder(mw).Encode(&micropubConfig{
MediaEndpoint: a.getFullAddress(micropubPath + micropubMediaSubPath),
})
_, _ = a.min.Write(w, contenttype.JSON, b)
case "source":
var mf interface{}
if urlString := r.URL.Query().Get("url"); urlString != "" {
@ -65,9 +65,9 @@ func (a *goBlog) serveMicropubQuery(w http.ResponseWriter, r *http.Request) {
mf = list
}
w.Header().Set(contentType, contenttype.JSONUTF8)
w.WriteHeader(http.StatusOK)
b, _ := json.Marshal(mf)
_, _ = a.min.Write(w, contenttype.JSON, b)
mw := a.min.Writer(contenttype.JSON, w)
defer mw.Close()
_ = json.NewEncoder(mw).Encode(mf)
case "category":
allCategories := []string{}
for blog := range a.cfg.Blogs {
@ -79,11 +79,11 @@ func (a *goBlog) serveMicropubQuery(w http.ResponseWriter, r *http.Request) {
allCategories = append(allCategories, values...)
}
w.Header().Set(contentType, contenttype.JSONUTF8)
w.WriteHeader(http.StatusOK)
b, _ := json.Marshal(map[string]interface{}{
mw := a.min.Writer(contenttype.JSON, w)
defer mw.Close()
_ = json.NewEncoder(mw).Encode(map[string]interface{}{
"categories": allCategories,
})
_, _ = a.min.Write(w, contenttype.JSON, b)
default:
a.serve404(w, r)
}

29
micropubMedia.go
View File

@ -1,43 +1,51 @@
package main
import (
"crypto/sha256"
"fmt"
"io"
"mime"
"net/http"
"path/filepath"
"strings"
"go.goblog.app/app/pkgs/bufferpool"
"go.goblog.app/app/pkgs/contenttype"
)
const micropubMediaSubPath = "/media"
func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) {
// Check scope
if !strings.Contains(r.Context().Value(indieAuthScope).(string), "media") {
a.serveError(w, r, "media scope missing", http.StatusForbidden)
return
}
// Check if request is multipart
if ct := r.Header.Get(contentType); !strings.Contains(ct, contenttype.MultipartForm) {
a.serveError(w, r, "wrong content-type", http.StatusBadRequest)
return
}
// Parse multipart form
err := r.ParseMultipartForm(0)
if err != nil {
a.serveError(w, r, err.Error(), http.StatusBadRequest)
return
}
// Get file
file, header, err := r.FormFile("file")
if err != nil {
a.serveError(w, r, err.Error(), http.StatusBadRequest)
return
}
defer func() { _ = file.Close() }()
hashFile, _, _ := r.FormFile("file")
defer func() { _ = hashFile.Close() }()
fileName, err := getSHA256(hashFile)
if err != nil {
a.serveError(w, r, err.Error(), http.StatusInternalServerError)
return
}
// Read the file into temporary buffer and generate sha256 hash
hash := sha256.New()
buffer := bufferpool.Get()
defer bufferpool.Put(buffer)
_, _ = io.Copy(buffer, io.TeeReader(file, hash))
_ = file.Close()
_ = r.Body.Close()
// Get file extension
fileExtension := filepath.Ext(header.Filename)
if len(fileExtension) == 0 {
// Find correct file extension if original filename does not contain one
@ -49,9 +57,10 @@ func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) {
}
}
}
fileName += strings.ToLower(fileExtension)
// Generate the file name
fileName := fmt.Sprintf("%x%s", hash.Sum(nil), fileExtension)
// Save file
location, err := a.saveMediaFile(fileName, file)
location, err := a.saveMediaFile(fileName, buffer)
if err != nil {
a.serveError(w, r, "failed to save original file: "+err.Error(), http.StatusInternalServerError)
return

14
nodeinfo.go
View File

@ -8,7 +8,10 @@ import (
)
func (a *goBlog) serveNodeInfoDiscover(w http.ResponseWriter, r *http.Request) {
b, _ := json.Marshal(map[string]interface{}{
w.Header().Set(contentType, contenttype.JSONUTF8)
mw := a.min.Writer(contenttype.JSON, w)
defer mw.Close()
_ = json.NewEncoder(mw).Encode(map[string]interface{}{
"links": []map[string]interface{}{
{
"href": a.getFullAddress("/nodeinfo"),
@ -16,15 +19,16 @@ func (a *goBlog) serveNodeInfoDiscover(w http.ResponseWriter, r *http.Request) {
},
},
})
w.Header().Set(contentType, contenttype.JSONUTF8)
_, _ = a.min.Write(w, contenttype.JSON, b)
}
func (a *goBlog) serveNodeInfo(w http.ResponseWriter, r *http.Request) {
localPosts, _ := a.db.countPosts(&postsRequestConfig{
status: statusPublished,
})
b, _ := json.Marshal(map[string]interface{}{
mw := a.min.Writer(contenttype.JSON, w)
defer mw.Close()
w.Header().Set(contentType, contenttype.JSONUTF8)
_ = json.NewEncoder(mw).Encode(map[string]interface{}{
"version": "2.1",
"software": map[string]interface{}{
"name": "goblog",
@ -43,6 +47,4 @@ func (a *goBlog) serveNodeInfo(w http.ResponseWriter, r *http.Request) {
},
"metadata": map[string]interface{}{},
})
w.Header().Set(contentType, contenttype.JSONUTF8)
_, _ = a.min.Write(w, contenttype.JSON, b)
}

18
pkgs/minify/minify.go
View File

@ -38,19 +38,19 @@ func (m *Minifier) Get() *minify.M {
}
func (m *Minifier) Write(w io.Writer, mediatype string, b []byte) (int, error) {
mw := m.Get().Writer(mediatype, w)
mw := m.Writer(mediatype, w)
defer mw.Close()
return mw.Write(b)
}
func (m *Minifier) MinifyBytes(mediatype string, b []byte) ([]byte, error) {
return m.Get().Bytes(mediatype, b)
}
func (m *Minifier) MinifyString(mediatype string, s string) (string, error) {
return m.Get().String(mediatype, s)
}
func (m *Minifier) Minify(mediatype string, w io.Writer, r io.Reader) error {
return m.Get().Minify(mediatype, w, r)
}
func (m *Minifier) Writer(mediatype string, w io.Writer) io.WriteCloser {
return m.Get().Writer(mediatype, w)
}
func (m *Minifier) Reader(mediatype string, r io.Reader) io.Reader {
return m.Get().Reader(mediatype, r)
}

6
render.go
View File

@ -40,10 +40,10 @@ func (a *goBlog) renderWithStatusCode(w http.ResponseWriter, r *http.Request, st
w.WriteHeader(statusCode)
// Render
buf := bufferpool.Get()
minWriter := a.min.Get().Writer(contenttype.HTML, buf)
hb := newHtmlBuilder(minWriter)
mw := a.min.Writer(contenttype.HTML, buf)
hb := newHtmlBuilder(mw)
f(hb, data)
_ = minWriter.Close()
_ = mw.Close()
_, _ = buf.WriteTo(w)
bufferpool.Put(buf)
}

31
templateAssets.go
View File

@ -2,6 +2,8 @@ package main
import (
"bytes"
"crypto/sha256"
"fmt"
"io"
"mime"
"net/http"
@ -55,37 +57,26 @@ func (a *goBlog) initTemplateAssets() error {
func (a *goBlog) compileAsset(name string, read io.Reader) (string, error) {
ext := path.Ext(name)
compiledExt := ext
var contentBuffer bytes.Buffer
switch ext {
case ".js":
if err := a.min.Minify(contenttype.JS, &contentBuffer, read); err != nil {
return "", err
}
read = a.min.Reader(contenttype.JS, read)
case ".css":
if err := a.min.Minify(contenttype.CSS, &contentBuffer, read); err != nil {
return "", err
}
read = a.min.Reader(contenttype.CSS, read)
case ".xml", ".xsl":
if err := a.min.Minify(contenttype.XML, &contentBuffer, read); err != nil {
return "", err
}
default:
if _, err := io.Copy(&contentBuffer, read); err != nil {
return "", err
}
read = a.min.Reader(contenttype.XML, read)
}
// Hashes
hash, err := getSHA256(bytes.NewReader(contentBuffer.Bytes()))
// Read file
hash := sha256.New()
body, err := io.ReadAll(io.TeeReader(read, hash))
if err != nil {
return "", err
}
// File name
compiledFileName := hash + compiledExt
compiledFileName := fmt.Sprintf("%x%s", hash.Sum(nil), ext)
// Create struct
a.assetFiles[compiledFileName] = &assetFile{
contentType: mime.TypeByExtension(compiledExt),
body: contentBuffer.Bytes(),
contentType: mime.TypeByExtension(ext),
body: body,
}
return compiledFileName, err
}

14
tts.go
View File

@ -3,8 +3,10 @@ package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"errors"
"fmt"
"html"
"io"
"log"
@ -105,18 +107,14 @@ func (a *goBlog) createPostTTSAudio(p *post) error {
}
// Merge partsBuffers into final buffer
var final bytes.Buffer
if err := mp3merge.MergeMP3(&final, partsBuffers...); err != nil {
final := new(bytes.Buffer)
hash := sha256.New()
if err := mp3merge.MergeMP3(io.MultiWriter(final, hash), partsBuffers...); err != nil {
return err
}
// Save audio
audioReader := bytes.NewReader(final.Bytes())
fileHash, err := getSHA256(audioReader)
if err != nil {
return err
}
loc, err := a.saveMediaFile(fileHash+".mp3", audioReader)
loc, err := a.saveMediaFile(fmt.Sprintf("%x.mp3", hash.Sum(nil)), final)
if err != nil {
return err
}

16
utils.go
View File

@ -2,7 +2,6 @@ package main
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
@ -217,21 +216,6 @@ func urlHasExt(rawUrl string, allowed ...string) (ext string, has bool) {
return ext, funk.ContainsString(allowed, strings.ToLower(ext))
}
// Get SHA-256 hash
func getSHA256(file io.ReadSeeker) (hash string, err error) {
if _, err = file.Seek(0, 0); err != nil {
return "", err
}
h := sha256.New()
if _, err = io.Copy(h, file); err != nil {
return "", err
}
if _, err = file.Seek(0, 0); err != nil {
return "", err
}
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
func mBytesString(size int64) string {
return fmt.Sprintf("%.2f MB", datasize.ByteSize(size).MBytes())
}