diff --git a/cache.go b/cache.go index 57be8b6..7f8de71 100644 --- a/cache.go +++ b/cache.go @@ -1,8 +1,8 @@ package main import ( - "bytes" "context" + "crypto/sha256" "encoding/binary" "fmt" "io" @@ -181,12 +181,9 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item * next.ServeHTTP(recorder, cr) // Cache values from recorder result := recorder.Result() - body, _ := io.ReadAll(result.Body) + eTag := sha256.New() + body, _ := io.ReadAll(io.TeeReader(result.Body, eTag)) _ = result.Body.Close() - eTag := result.Header.Get("ETag") - if eTag == "" { - eTag, _ = getSHA256(bytes.NewReader(body)) - } lastMod := time.Now() if lm := result.Header.Get(lastModified); lm != "" { if parsedTime, te := dateparse.ParseLocal(lm); te == nil { @@ -202,7 +199,7 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item * item = &cacheItem{ expiration: exp, creationTime: lastMod, - eTag: eTag, + eTag: fmt.Sprintf("%x", eTag.Sum(nil)), code: result.StatusCode, header: result.Header, body: body, diff --git a/editor.go b/editor.go index 36e5f40..2b850a2 100644 --- a/editor.go +++ b/editor.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "context" "encoding/json" "fmt" @@ -45,37 +44,34 @@ func (a *goBlog) serveEditorPreview(w http.ResponseWriter, r *http.Request) { continue } // Create preview - preview, err := a.createMarkdownPreview(blog, string(message)) + w, err := c.Writer(ctx, ws.MessageText) if err != nil { - preview = []byte(err.Error()) + break } - // Write preview to socket - err = c.Write(ctx, ws.MessageText, preview) + a.createMarkdownPreview(w, blog, string(message)) + err = w.Close() if err != nil { break } } } -func (a *goBlog) createMarkdownPreview(blog string, markdown string) (rendered []byte, err error) { +func (a *goBlog) createMarkdownPreview(w io.Writer, blog string, markdown string) { p := &post{ Blog: blog, Content: markdown, } - err = a.computeExtraPostParameters(p) + err := a.computeExtraPostParameters(p) if err != nil { - return nil, err + _, _ = io.WriteString(w, err.Error()) + return } if t := p.Title(); t != "" { p.RenderedTitle = a.renderMdTitle(t) } // Render post - buf := bufferpool.Get() - hb := newHtmlBuilder(buf) + hb := newHtmlBuilder(w) a.renderEditorPreview(hb, a.cfg.Blogs[blog], p) - rendered = buf.Bytes() - bufferpool.Put(buf) - return } func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) { @@ -94,7 +90,9 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) { }, }) case "updatepost": - jsonBytes, err := json.Marshal(map[string]interface{}{ + jsonBuf := bufferpool.Get() + defer bufferpool.Put(jsonBuf) + err := json.NewEncoder(jsonBuf).Encode(map[string]interface{}{ "action": actionUpdate, "url": r.FormValue("url"), "replace": map[string][]string{ @@ -107,7 +105,7 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) { a.serveError(w, r, err.Error(), http.StatusInternalServerError) return } - req, err := http.NewRequest(http.MethodPost, "", bytes.NewReader(jsonBytes)) + req, err := http.NewRequest(http.MethodPost, "", jsonBuf) if err != nil { a.serveError(w, r, err.Error(), http.StatusInternalServerError) return @@ -138,25 +136,15 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) { a.serveError(w, r, err.Error(), http.StatusBadRequest) return } - originalGpx, err := io.ReadAll(file) - if err != nil { - a.serveError(w, r, err.Error(), http.StatusBadRequest) - return - } - minifiedGpx, err := a.min.MinifyString(contenttype.XML, string(originalGpx)) - if err != nil { - a.serveError(w, r, err.Error(), http.StatusInternalServerError) - return - } - resultBytes, err := yaml.Marshal(map[string]string{ - "gpx": minifiedGpx, - }) + gpx, err := io.ReadAll(a.min.Reader(contenttype.XML, file)) if err != nil { a.serveError(w, r, err.Error(), http.StatusBadRequest) return } w.Header().Set(contentType, contenttype.TextUTF8) - _, _ = w.Write(resultBytes) + _ = yaml.NewEncoder(w).Encode(map[string]string{ + "gpx": string(gpx), + }) default: a.serveError(w, r, "Unknown editoraction", http.StatusBadRequest) } diff --git a/httpFs.go b/httpFs.go index a49b953..d839ca7 100644 --- a/httpFs.go +++ b/httpFs.go @@ -1,7 +1,8 @@ package main import ( - "embed" + "io" + "io/fs" "net/http" "path" "strings" @@ -9,24 +10,23 @@ import ( "go.goblog.app/app/pkgs/contenttype" ) -func (a *goBlog) serveFs(fs embed.FS, basePath string) http.HandlerFunc { +func (a *goBlog) serveFs(f fs.FS, basePath string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { fileName := strings.TrimPrefix(r.URL.Path, basePath) - fb, err := fs.ReadFile(fileName) + file, err := f.Open(fileName) if err != nil { a.serve404(w, r) return } + var read io.Reader = file switch path.Ext(fileName) { case ".js": - w.Header().Set(contentType, contenttype.JS) - _, _ = a.min.Write(w, contenttype.JSUTF8, fb) + w.Header().Set(contentType, contenttype.JSUTF8) + read = a.min.Reader(contenttype.JS, read) case ".css": - w.Header().Set(contentType, contenttype.CSS) - _, _ = a.min.Write(w, contenttype.CSSUTF8, fb) - default: - w.Header().Set(contentType, http.DetectContentType(fb)) - _, _ = w.Write(fb) + w.Header().Set(contentType, contenttype.CSSUTF8) + read = a.min.Reader(contenttype.CSS, read) } + _, _ = io.Copy(w, read) } } diff --git a/mediaCompression.go b/mediaCompression.go index 2a43f25..e9ae634 100644 --- a/mediaCompression.go +++ b/mediaCompression.go @@ -3,12 +3,15 @@ package main import ( "bytes" "context" + "crypto/sha256" "errors" "fmt" + "io" "log" "net/http" "github.com/carlmjohnson/requests" + "go.goblog.app/app/pkgs/bufferpool" ) const defaultCompressionWidth = 2000 @@ -167,14 +170,12 @@ func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http return uploadCompressedFile(fileExtension, &imgBuffer, upload) } -func uploadCompressedFile(fileExtension string, imgBuffer *bytes.Buffer, upload mediaStorageSaveFunc) (string, error) { - // Create reader from buffer - imgReader := bytes.NewReader(imgBuffer.Bytes()) - // Get hash of compressed file - fileName, err := getSHA256(imgReader) - if err != nil { - return "", err - } - // Upload compressed file - return upload(fileName+"."+fileExtension, imgReader) +func uploadCompressedFile(fileExtension string, r io.Reader, upload mediaStorageSaveFunc) (string, error) { + // Copy file to temporary buffer to generate hash and filename + hash := sha256.New() + tempBuffer := bufferpool.Get() + defer bufferpool.Put(tempBuffer) + _, _ = io.Copy(io.MultiWriter(tempBuffer, hash), r) + // Upload buffer + return upload(fmt.Sprintf("%x.%s", hash.Sum(nil), fileExtension), tempBuffer) } diff --git a/mediaCompression_test.go b/mediaCompression_test.go index df7112f..ac2d165 100644 --- a/mediaCompression_test.go +++ b/mediaCompression_test.go @@ -1,11 +1,11 @@ package main import ( + "crypto/sha256" "encoding/json" + "fmt" "io" "net/http" - "os" - "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -14,13 +14,9 @@ import ( func Test_compress(t *testing.T) { fakeFileContent := "Test" - fakeFileName := filepath.Join(t.TempDir(), "test.jpg") - err := os.WriteFile(fakeFileName, []byte(fakeFileContent), 0666) - require.Nil(t, err) - fakeFile, err := os.Open(fakeFileName) - require.Nil(t, err) - fakeSha256, err := getSHA256(fakeFile) - require.Nil(t, err) + hash := sha256.New() + io.WriteString(hash, fakeFileContent) + fakeSha256 := fmt.Sprintf("%x", hash.Sum(nil)) var uf mediaStorageSaveFunc = func(filename string, f io.Reader) (location string, err error) { return "https://example.com/" + filename, nil @@ -32,7 +28,7 @@ func Test_compress(t *testing.T) { assert.Equal(t, "https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=2000,h=3000/https://example.com/original.jpg", r.URL.String()) rw.WriteHeader(http.StatusOK) - _, _ = rw.Write([]byte(fakeFileContent)) + _, _ = io.WriteString(rw, fakeFileContent) })) cf := &cloudflare{} @@ -51,7 +47,7 @@ func Test_compress(t *testing.T) { defer r.Body.Close() var requestJson map[string]interface{} - err = json.Unmarshal(requestBody, &requestJson) + err := json.Unmarshal(requestBody, &requestJson) require.Nil(t, err) require.NotNil(t, requestJson) @@ -59,7 +55,7 @@ func Test_compress(t *testing.T) { assert.Equal(t, "https://example.com/original.jpg", requestJson["url"]) rw.WriteHeader(http.StatusOK) - _, _ = rw.Write([]byte(fakeFileContent)) + _, _ = io.WriteString(rw, fakeFileContent) })) cf := &shortpixel{"testkey"} diff --git a/micropub.go b/micropub.go index c7838a6..1fe73c1 100644 --- a/micropub.go +++ b/micropub.go @@ -28,11 +28,11 @@ func (a *goBlog) serveMicropubQuery(w http.ResponseWriter, r *http.Request) { switch r.URL.Query().Get("q") { case "config": w.Header().Set(contentType, contenttype.JSONUTF8) - w.WriteHeader(http.StatusOK) - b, _ := json.Marshal(µpubConfig{ + mw := a.min.Writer(contenttype.JSON, w) + defer mw.Close() + _ = json.NewEncoder(mw).Encode(µpubConfig{ MediaEndpoint: a.getFullAddress(micropubPath + micropubMediaSubPath), }) - _, _ = a.min.Write(w, contenttype.JSON, b) case "source": var mf interface{} if urlString := r.URL.Query().Get("url"); urlString != "" { @@ -65,9 +65,9 @@ func (a *goBlog) serveMicropubQuery(w http.ResponseWriter, r *http.Request) { mf = list } w.Header().Set(contentType, contenttype.JSONUTF8) - w.WriteHeader(http.StatusOK) - b, _ := json.Marshal(mf) - _, _ = a.min.Write(w, contenttype.JSON, b) + mw := a.min.Writer(contenttype.JSON, w) + defer mw.Close() + _ = json.NewEncoder(mw).Encode(mf) case "category": allCategories := []string{} for blog := range a.cfg.Blogs { @@ -79,11 +79,11 @@ func (a *goBlog) serveMicropubQuery(w http.ResponseWriter, r *http.Request) { allCategories = append(allCategories, values...) } w.Header().Set(contentType, contenttype.JSONUTF8) - w.WriteHeader(http.StatusOK) - b, _ := json.Marshal(map[string]interface{}{ + mw := a.min.Writer(contenttype.JSON, w) + defer mw.Close() + _ = json.NewEncoder(mw).Encode(map[string]interface{}{ "categories": allCategories, }) - _, _ = a.min.Write(w, contenttype.JSON, b) default: a.serve404(w, r) } diff --git a/micropubMedia.go b/micropubMedia.go index 05b542a..1540a5d 100644 --- a/micropubMedia.go +++ b/micropubMedia.go @@ -1,43 +1,51 @@ package main import ( + "crypto/sha256" + "fmt" + "io" "mime" "net/http" "path/filepath" "strings" + "go.goblog.app/app/pkgs/bufferpool" "go.goblog.app/app/pkgs/contenttype" ) const micropubMediaSubPath = "/media" func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) { + // Check scope if !strings.Contains(r.Context().Value(indieAuthScope).(string), "media") { a.serveError(w, r, "media scope missing", http.StatusForbidden) return } + // Check if request is multipart if ct := r.Header.Get(contentType); !strings.Contains(ct, contenttype.MultipartForm) { a.serveError(w, r, "wrong content-type", http.StatusBadRequest) return } + // Parse multipart form err := r.ParseMultipartForm(0) if err != nil { a.serveError(w, r, err.Error(), http.StatusBadRequest) return } + // Get file file, header, err := r.FormFile("file") if err != nil { a.serveError(w, r, err.Error(), http.StatusBadRequest) return } - defer func() { _ = file.Close() }() - hashFile, _, _ := r.FormFile("file") - defer func() { _ = hashFile.Close() }() - fileName, err := getSHA256(hashFile) - if err != nil { - a.serveError(w, r, err.Error(), http.StatusInternalServerError) - return - } + // Read the file into temporary buffer and generate sha256 hash + hash := sha256.New() + buffer := bufferpool.Get() + defer bufferpool.Put(buffer) + _, _ = io.Copy(buffer, io.TeeReader(file, hash)) + _ = file.Close() + _ = r.Body.Close() + // Get file extension fileExtension := filepath.Ext(header.Filename) if len(fileExtension) == 0 { // Find correct file extension if original filename does not contain one @@ -49,9 +57,10 @@ func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) { } } } - fileName += strings.ToLower(fileExtension) + // Generate the file name + fileName := fmt.Sprintf("%x%s", hash.Sum(nil), fileExtension) // Save file - location, err := a.saveMediaFile(fileName, file) + location, err := a.saveMediaFile(fileName, buffer) if err != nil { a.serveError(w, r, "failed to save original file: "+err.Error(), http.StatusInternalServerError) return diff --git a/nodeinfo.go b/nodeinfo.go index 27de9c4..144e86b 100644 --- a/nodeinfo.go +++ b/nodeinfo.go @@ -8,7 +8,10 @@ import ( ) func (a *goBlog) serveNodeInfoDiscover(w http.ResponseWriter, r *http.Request) { - b, _ := json.Marshal(map[string]interface{}{ + w.Header().Set(contentType, contenttype.JSONUTF8) + mw := a.min.Writer(contenttype.JSON, w) + defer mw.Close() + _ = json.NewEncoder(mw).Encode(map[string]interface{}{ "links": []map[string]interface{}{ { "href": a.getFullAddress("/nodeinfo"), @@ -16,15 +19,16 @@ func (a *goBlog) serveNodeInfoDiscover(w http.ResponseWriter, r *http.Request) { }, }, }) - w.Header().Set(contentType, contenttype.JSONUTF8) - _, _ = a.min.Write(w, contenttype.JSON, b) } func (a *goBlog) serveNodeInfo(w http.ResponseWriter, r *http.Request) { localPosts, _ := a.db.countPosts(&postsRequestConfig{ status: statusPublished, }) - b, _ := json.Marshal(map[string]interface{}{ + mw := a.min.Writer(contenttype.JSON, w) + defer mw.Close() + w.Header().Set(contentType, contenttype.JSONUTF8) + _ = json.NewEncoder(mw).Encode(map[string]interface{}{ "version": "2.1", "software": map[string]interface{}{ "name": "goblog", @@ -43,6 +47,4 @@ func (a *goBlog) serveNodeInfo(w http.ResponseWriter, r *http.Request) { }, "metadata": map[string]interface{}{}, }) - w.Header().Set(contentType, contenttype.JSONUTF8) - _, _ = a.min.Write(w, contenttype.JSON, b) } diff --git a/pkgs/minify/minify.go b/pkgs/minify/minify.go index 54830eb..2c0c11c 100644 --- a/pkgs/minify/minify.go +++ b/pkgs/minify/minify.go @@ -38,19 +38,19 @@ func (m *Minifier) Get() *minify.M { } func (m *Minifier) Write(w io.Writer, mediatype string, b []byte) (int, error) { - mw := m.Get().Writer(mediatype, w) + mw := m.Writer(mediatype, w) defer mw.Close() return mw.Write(b) } -func (m *Minifier) MinifyBytes(mediatype string, b []byte) ([]byte, error) { - return m.Get().Bytes(mediatype, b) -} - -func (m *Minifier) MinifyString(mediatype string, s string) (string, error) { - return m.Get().String(mediatype, s) -} - func (m *Minifier) Minify(mediatype string, w io.Writer, r io.Reader) error { return m.Get().Minify(mediatype, w, r) } + +func (m *Minifier) Writer(mediatype string, w io.Writer) io.WriteCloser { + return m.Get().Writer(mediatype, w) +} + +func (m *Minifier) Reader(mediatype string, r io.Reader) io.Reader { + return m.Get().Reader(mediatype, r) +} diff --git a/render.go b/render.go index c88a670..f6e5c12 100644 --- a/render.go +++ b/render.go @@ -40,10 +40,10 @@ func (a *goBlog) renderWithStatusCode(w http.ResponseWriter, r *http.Request, st w.WriteHeader(statusCode) // Render buf := bufferpool.Get() - minWriter := a.min.Get().Writer(contenttype.HTML, buf) - hb := newHtmlBuilder(minWriter) + mw := a.min.Writer(contenttype.HTML, buf) + hb := newHtmlBuilder(mw) f(hb, data) - _ = minWriter.Close() + _ = mw.Close() _, _ = buf.WriteTo(w) bufferpool.Put(buf) } diff --git a/templateAssets.go b/templateAssets.go index 795a43a..2b31eba 100644 --- a/templateAssets.go +++ b/templateAssets.go @@ -2,6 +2,8 @@ package main import ( "bytes" + "crypto/sha256" + "fmt" "io" "mime" "net/http" @@ -55,37 +57,26 @@ func (a *goBlog) initTemplateAssets() error { func (a *goBlog) compileAsset(name string, read io.Reader) (string, error) { ext := path.Ext(name) - compiledExt := ext - var contentBuffer bytes.Buffer switch ext { case ".js": - if err := a.min.Minify(contenttype.JS, &contentBuffer, read); err != nil { - return "", err - } + read = a.min.Reader(contenttype.JS, read) case ".css": - if err := a.min.Minify(contenttype.CSS, &contentBuffer, read); err != nil { - return "", err - } + read = a.min.Reader(contenttype.CSS, read) case ".xml", ".xsl": - if err := a.min.Minify(contenttype.XML, &contentBuffer, read); err != nil { - return "", err - } - default: - if _, err := io.Copy(&contentBuffer, read); err != nil { - return "", err - } + read = a.min.Reader(contenttype.XML, read) } - // Hashes - hash, err := getSHA256(bytes.NewReader(contentBuffer.Bytes())) + // Read file + hash := sha256.New() + body, err := io.ReadAll(io.TeeReader(read, hash)) if err != nil { return "", err } // File name - compiledFileName := hash + compiledExt + compiledFileName := fmt.Sprintf("%x%s", hash.Sum(nil), ext) // Create struct a.assetFiles[compiledFileName] = &assetFile{ - contentType: mime.TypeByExtension(compiledExt), - body: contentBuffer.Bytes(), + contentType: mime.TypeByExtension(ext), + body: body, } return compiledFileName, err } diff --git a/tts.go b/tts.go index 638a7af..3ebce24 100644 --- a/tts.go +++ b/tts.go @@ -3,8 +3,10 @@ package main import ( "bytes" "context" + "crypto/sha256" "encoding/base64" "errors" + "fmt" "html" "io" "log" @@ -105,18 +107,14 @@ func (a *goBlog) createPostTTSAudio(p *post) error { } // Merge partsBuffers into final buffer - var final bytes.Buffer - if err := mp3merge.MergeMP3(&final, partsBuffers...); err != nil { + final := new(bytes.Buffer) + hash := sha256.New() + if err := mp3merge.MergeMP3(io.MultiWriter(final, hash), partsBuffers...); err != nil { return err } // Save audio - audioReader := bytes.NewReader(final.Bytes()) - fileHash, err := getSHA256(audioReader) - if err != nil { - return err - } - loc, err := a.saveMediaFile(fileHash+".mp3", audioReader) + loc, err := a.saveMediaFile(fmt.Sprintf("%x.mp3", hash.Sum(nil)), final) if err != nil { return err } diff --git a/utils.go b/utils.go index 0dad579..5a28a3e 100644 --- a/utils.go +++ b/utils.go @@ -2,7 +2,6 @@ package main import ( "context" - "crypto/sha256" "errors" "fmt" "io" @@ -217,21 +216,6 @@ func urlHasExt(rawUrl string, allowed ...string) (ext string, has bool) { return ext, funk.ContainsString(allowed, strings.ToLower(ext)) } -// Get SHA-256 hash -func getSHA256(file io.ReadSeeker) (hash string, err error) { - if _, err = file.Seek(0, 0); err != nil { - return "", err - } - h := sha256.New() - if _, err = io.Copy(h, file); err != nil { - return "", err - } - if _, err = file.Seek(0, 0); err != nil { - return "", err - } - return fmt.Sprintf("%x", h.Sum(nil)), nil -} - func mBytesString(size int64) string { return fmt.Sprintf("%.2f MB", datasize.ByteSize(size).MBytes()) }