More io streaming

cache.go

@@ -1,8 +1,8 @@
 package main
 
 import (
-	"bytes"
 	"context"
+	"crypto/sha256"
 	"encoding/binary"
 	"fmt"
 	"io"
@@ -181,12 +181,9 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *
 		next.ServeHTTP(recorder, cr)
 		// Cache values from recorder
 		result := recorder.Result()
-		body, _ := io.ReadAll(result.Body)
+		eTag := sha256.New()
+		body, _ := io.ReadAll(io.TeeReader(result.Body, eTag))
 		_ = result.Body.Close()
-		eTag := result.Header.Get("ETag")
-		if eTag == "" {
-			eTag, _ = getSHA256(bytes.NewReader(body))
-		}
 		lastMod := time.Now()
 		if lm := result.Header.Get(lastModified); lm != "" {
 			if parsedTime, te := dateparse.ParseLocal(lm); te == nil {
@@ -202,7 +199,7 @@ func (c *cache) getCache(key string, next http.Handler, r *http.Request) (item *
 		item = &cacheItem{
 			expiration:   exp,
 			creationTime: lastMod,
-			eTag:         eTag,
+			eTag:         fmt.Sprintf("%x", eTag.Sum(nil)),
 			code:         result.StatusCode,
 			header:       result.Header,
 			body:         body,

editor.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -45,37 +44,34 @@ func (a *goBlog) serveEditorPreview(w http.ResponseWriter, r *http.Request) {
 			continue
 		}
 		// Create preview
-		preview, err := a.createMarkdownPreview(blog, string(message))
+		w, err := c.Writer(ctx, ws.MessageText)
 		if err != nil {
-			preview = []byte(err.Error())
+			break
 		}
-		// Write preview to socket
+		a.createMarkdownPreview(w, blog, string(message))
-		err = c.Write(ctx, ws.MessageText, preview)
+		err = w.Close()
 		if err != nil {
 			break
 		}
 	}
 }
 
-func (a *goBlog) createMarkdownPreview(blog string, markdown string) (rendered []byte, err error) {
+func (a *goBlog) createMarkdownPreview(w io.Writer, blog string, markdown string) {
 	p := &post{
 		Blog:    blog,
 		Content: markdown,
 	}
-	err = a.computeExtraPostParameters(p)
+	err := a.computeExtraPostParameters(p)
 	if err != nil {
-		return nil, err
+		_, _ = io.WriteString(w, err.Error())
+		return
 	}
 	if t := p.Title(); t != "" {
 		p.RenderedTitle = a.renderMdTitle(t)
 	}
 	// Render post
-	buf := bufferpool.Get()
+	hb := newHtmlBuilder(w)
-	hb := newHtmlBuilder(buf)
 	a.renderEditorPreview(hb, a.cfg.Blogs[blog], p)
-	rendered = buf.Bytes()
-	bufferpool.Put(buf)
-	return
 }
 
 func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) {
@@ -94,7 +90,9 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) {
 				},
 			})
 		case "updatepost":
-			jsonBytes, err := json.Marshal(map[string]interface{}{
+			jsonBuf := bufferpool.Get()
+			defer bufferpool.Put(jsonBuf)
+			err := json.NewEncoder(jsonBuf).Encode(map[string]interface{}{
 				"action": actionUpdate,
 				"url":    r.FormValue("url"),
 				"replace": map[string][]string{
@@ -107,7 +105,7 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) {
 				a.serveError(w, r, err.Error(), http.StatusInternalServerError)
 				return
 			}
-			req, err := http.NewRequest(http.MethodPost, "", bytes.NewReader(jsonBytes))
+			req, err := http.NewRequest(http.MethodPost, "", jsonBuf)
 			if err != nil {
 				a.serveError(w, r, err.Error(), http.StatusInternalServerError)
 				return
@@ -138,25 +136,15 @@ func (a *goBlog) serveEditorPost(w http.ResponseWriter, r *http.Request) {
 				a.serveError(w, r, err.Error(), http.StatusBadRequest)
 				return
 			}
-			originalGpx, err := io.ReadAll(file)
+			gpx, err := io.ReadAll(a.min.Reader(contenttype.XML, file))
-			if err != nil {
-				a.serveError(w, r, err.Error(), http.StatusBadRequest)
-				return
-			}
-			minifiedGpx, err := a.min.MinifyString(contenttype.XML, string(originalGpx))
-			if err != nil {
-				a.serveError(w, r, err.Error(), http.StatusInternalServerError)
-				return
-			}
-			resultBytes, err := yaml.Marshal(map[string]string{
-				"gpx": minifiedGpx,
-			})
 			if err != nil {
 				a.serveError(w, r, err.Error(), http.StatusBadRequest)
 				return
 			}
 			w.Header().Set(contentType, contenttype.TextUTF8)
-			_, _ = w.Write(resultBytes)
+			_ = yaml.NewEncoder(w).Encode(map[string]string{
+				"gpx": string(gpx),
+			})
 		default:
 			a.serveError(w, r, "Unknown editoraction", http.StatusBadRequest)
 		}

httpFs.go

@@ -1,7 +1,8 @@
 package main
 
 import (
-	"embed"
+	"io"
+	"io/fs"
 	"net/http"
 	"path"
 	"strings"
@@ -9,24 +10,23 @@ import (
 	"go.goblog.app/app/pkgs/contenttype"
 )
 
-func (a *goBlog) serveFs(fs embed.FS, basePath string) http.HandlerFunc {
+func (a *goBlog) serveFs(f fs.FS, basePath string) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		fileName := strings.TrimPrefix(r.URL.Path, basePath)
-		fb, err := fs.ReadFile(fileName)
+		file, err := f.Open(fileName)
 		if err != nil {
 			a.serve404(w, r)
 			return
 		}
+		var read io.Reader = file
 		switch path.Ext(fileName) {
 		case ".js":
-			w.Header().Set(contentType, contenttype.JS)
+			w.Header().Set(contentType, contenttype.JSUTF8)
-			_, _ = a.min.Write(w, contenttype.JSUTF8, fb)
+			read = a.min.Reader(contenttype.JS, read)
 		case ".css":
-			w.Header().Set(contentType, contenttype.CSS)
+			w.Header().Set(contentType, contenttype.CSSUTF8)
-			_, _ = a.min.Write(w, contenttype.CSSUTF8, fb)
+			read = a.min.Reader(contenttype.CSS, read)
-		default:
-			w.Header().Set(contentType, http.DetectContentType(fb))
-			_, _ = w.Write(fb)
 		}
+		_, _ = io.Copy(w, read)
 	}
 }

mediaCompression.go

@@ -3,12 +3,15 @@ package main
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
 	"errors"
 	"fmt"
+	"io"
 	"log"
 	"net/http"
 
 	"github.com/carlmjohnson/requests"
+	"go.goblog.app/app/pkgs/bufferpool"
 )
 
 const defaultCompressionWidth = 2000
@@ -167,14 +170,12 @@ func (cf *cloudflare) compress(url string, upload mediaStorageSaveFunc, hc *http
 	return uploadCompressedFile(fileExtension, &imgBuffer, upload)
 }
 
-func uploadCompressedFile(fileExtension string, imgBuffer *bytes.Buffer, upload mediaStorageSaveFunc) (string, error) {
+func uploadCompressedFile(fileExtension string, r io.Reader, upload mediaStorageSaveFunc) (string, error) {
-	// Create reader from buffer
+	// Copy file to temporary buffer to generate hash and filename
-	imgReader := bytes.NewReader(imgBuffer.Bytes())
+	hash := sha256.New()
-	// Get hash of compressed file
+	tempBuffer := bufferpool.Get()
-	fileName, err := getSHA256(imgReader)
+	defer bufferpool.Put(tempBuffer)
-	if err != nil {
+	_, _ = io.Copy(io.MultiWriter(tempBuffer, hash), r)
-		return "", err
+	// Upload buffer
-	}
+	return upload(fmt.Sprintf("%x.%s", hash.Sum(nil), fileExtension), tempBuffer)
-	// Upload compressed file
-	return upload(fileName+"."+fileExtension, imgReader)
 }

mediaCompression_test.go

@@ -1,11 +1,11 @@
 package main
 
 import (
+	"crypto/sha256"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
-	"os"
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -14,13 +14,9 @@ import (
 
 func Test_compress(t *testing.T) {
 	fakeFileContent := "Test"
-	fakeFileName := filepath.Join(t.TempDir(), "test.jpg")
+	hash := sha256.New()
-	err := os.WriteFile(fakeFileName, []byte(fakeFileContent), 0666)
+	io.WriteString(hash, fakeFileContent)
-	require.Nil(t, err)
+	fakeSha256 := fmt.Sprintf("%x", hash.Sum(nil))
-	fakeFile, err := os.Open(fakeFileName)
-	require.Nil(t, err)
-	fakeSha256, err := getSHA256(fakeFile)
-	require.Nil(t, err)
 
 	var uf mediaStorageSaveFunc = func(filename string, f io.Reader) (location string, err error) {
 		return "https://example.com/" + filename, nil
@@ -32,7 +28,7 @@ func Test_compress(t *testing.T) {
 			assert.Equal(t, "https://www.cloudflare.com/cdn-cgi/image/f=jpeg,q=75,metadata=none,fit=scale-down,w=2000,h=3000/https://example.com/original.jpg", r.URL.String())
 
 			rw.WriteHeader(http.StatusOK)
-			_, _ = rw.Write([]byte(fakeFileContent))
+			_, _ = io.WriteString(rw, fakeFileContent)
 		}))
 
 		cf := &cloudflare{}
@@ -51,7 +47,7 @@ func Test_compress(t *testing.T) {
 			defer r.Body.Close()
 
 			var requestJson map[string]interface{}
-			err = json.Unmarshal(requestBody, &requestJson)
+			err := json.Unmarshal(requestBody, &requestJson)
 			require.Nil(t, err)
 			require.NotNil(t, requestJson)
 
@@ -59,7 +55,7 @@ func Test_compress(t *testing.T) {
 			assert.Equal(t, "https://example.com/original.jpg", requestJson["url"])
 
 			rw.WriteHeader(http.StatusOK)
-			_, _ = rw.Write([]byte(fakeFileContent))
+			_, _ = io.WriteString(rw, fakeFileContent)
 		}))
 
 		cf := &shortpixel{"testkey"}

micropub.go

@@ -28,11 +28,11 @@ func (a *goBlog) serveMicropubQuery(w http.ResponseWriter, r *http.Request) {
 	switch r.URL.Query().Get("q") {
 	case "config":
 		w.Header().Set(contentType, contenttype.JSONUTF8)
-		w.WriteHeader(http.StatusOK)
+		mw := a.min.Writer(contenttype.JSON, w)
-		b, _ := json.Marshal(&micropubConfig{
+		defer mw.Close()
+		_ = json.NewEncoder(mw).Encode(&micropubConfig{
 			MediaEndpoint: a.getFullAddress(micropubPath + micropubMediaSubPath),
 		})
-		_, _ = a.min.Write(w, contenttype.JSON, b)
 	case "source":
 		var mf interface{}
 		if urlString := r.URL.Query().Get("url"); urlString != "" {
@@ -65,9 +65,9 @@ func (a *goBlog) serveMicropubQuery(w http.ResponseWriter, r *http.Request) {
 			mf = list
 		}
 		w.Header().Set(contentType, contenttype.JSONUTF8)
-		w.WriteHeader(http.StatusOK)
+		mw := a.min.Writer(contenttype.JSON, w)
-		b, _ := json.Marshal(mf)
+		defer mw.Close()
-		_, _ = a.min.Write(w, contenttype.JSON, b)
+		_ = json.NewEncoder(mw).Encode(mf)
 	case "category":
 		allCategories := []string{}
 		for blog := range a.cfg.Blogs {
@@ -79,11 +79,11 @@ func (a *goBlog) serveMicropubQuery(w http.ResponseWriter, r *http.Request) {
 			allCategories = append(allCategories, values...)
 		}
 		w.Header().Set(contentType, contenttype.JSONUTF8)
-		w.WriteHeader(http.StatusOK)
+		mw := a.min.Writer(contenttype.JSON, w)
-		b, _ := json.Marshal(map[string]interface{}{
+		defer mw.Close()
+		_ = json.NewEncoder(mw).Encode(map[string]interface{}{
 			"categories": allCategories,
 		})
-		_, _ = a.min.Write(w, contenttype.JSON, b)
 	default:
 		a.serve404(w, r)
 	}

micropubMedia.go

@@ -1,43 +1,51 @@
 package main
 
 import (
+	"crypto/sha256"
+	"fmt"
+	"io"
 	"mime"
 	"net/http"
 	"path/filepath"
 	"strings"
 
+	"go.goblog.app/app/pkgs/bufferpool"
 	"go.goblog.app/app/pkgs/contenttype"
 )
 
 const micropubMediaSubPath = "/media"
 
 func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) {
+	// Check scope
 	if !strings.Contains(r.Context().Value(indieAuthScope).(string), "media") {
 		a.serveError(w, r, "media scope missing", http.StatusForbidden)
 		return
 	}
+	// Check if request is multipart
 	if ct := r.Header.Get(contentType); !strings.Contains(ct, contenttype.MultipartForm) {
 		a.serveError(w, r, "wrong content-type", http.StatusBadRequest)
 		return
 	}
+	// Parse multipart form
 	err := r.ParseMultipartForm(0)
 	if err != nil {
 		a.serveError(w, r, err.Error(), http.StatusBadRequest)
 		return
 	}
+	// Get file
 	file, header, err := r.FormFile("file")
 	if err != nil {
 		a.serveError(w, r, err.Error(), http.StatusBadRequest)
 		return
 	}
-	defer func() { _ = file.Close() }()
+	// Read the file into temporary buffer and generate sha256 hash
-	hashFile, _, _ := r.FormFile("file")
+	hash := sha256.New()
-	defer func() { _ = hashFile.Close() }()
+	buffer := bufferpool.Get()
-	fileName, err := getSHA256(hashFile)
+	defer bufferpool.Put(buffer)
-	if err != nil {
+	_, _ = io.Copy(buffer, io.TeeReader(file, hash))
-		a.serveError(w, r, err.Error(), http.StatusInternalServerError)
+	_ = file.Close()
-		return
+	_ = r.Body.Close()
-	}
+	// Get file extension
 	fileExtension := filepath.Ext(header.Filename)
 	if len(fileExtension) == 0 {
 		// Find correct file extension if original filename does not contain one
@@ -49,9 +57,10 @@ func (a *goBlog) serveMicropubMedia(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 	}
-	fileName += strings.ToLower(fileExtension)
+	// Generate the file name
+	fileName := fmt.Sprintf("%x%s", hash.Sum(nil), fileExtension)
 	// Save file
-	location, err := a.saveMediaFile(fileName, file)
+	location, err := a.saveMediaFile(fileName, buffer)
 	if err != nil {
 		a.serveError(w, r, "failed to save original file: "+err.Error(), http.StatusInternalServerError)
 		return

nodeinfo.go

@@ -8,7 +8,10 @@ import (
 )
 
 func (a *goBlog) serveNodeInfoDiscover(w http.ResponseWriter, r *http.Request) {
-	b, _ := json.Marshal(map[string]interface{}{
+	w.Header().Set(contentType, contenttype.JSONUTF8)
+	mw := a.min.Writer(contenttype.JSON, w)
+	defer mw.Close()
+	_ = json.NewEncoder(mw).Encode(map[string]interface{}{
 		"links": []map[string]interface{}{
 			{
 				"href": a.getFullAddress("/nodeinfo"),
@@ -16,15 +19,16 @@ func (a *goBlog) serveNodeInfoDiscover(w http.ResponseWriter, r *http.Request) {
 			},
 		},
 	})
-	w.Header().Set(contentType, contenttype.JSONUTF8)
-	_, _ = a.min.Write(w, contenttype.JSON, b)
 }
 
 func (a *goBlog) serveNodeInfo(w http.ResponseWriter, r *http.Request) {
 	localPosts, _ := a.db.countPosts(&postsRequestConfig{
 		status: statusPublished,
 	})
-	b, _ := json.Marshal(map[string]interface{}{
+	mw := a.min.Writer(contenttype.JSON, w)
+	defer mw.Close()
+	w.Header().Set(contentType, contenttype.JSONUTF8)
+	_ = json.NewEncoder(mw).Encode(map[string]interface{}{
 		"version": "2.1",
 		"software": map[string]interface{}{
 			"name":       "goblog",
@@ -43,6 +47,4 @@ func (a *goBlog) serveNodeInfo(w http.ResponseWriter, r *http.Request) {
 		},
 		"metadata": map[string]interface{}{},
 	})
-	w.Header().Set(contentType, contenttype.JSONUTF8)
-	_, _ = a.min.Write(w, contenttype.JSON, b)
 }

pkgs/minify/minify.go

@@ -38,19 +38,19 @@ func (m *Minifier) Get() *minify.M {
 }
 
 func (m *Minifier) Write(w io.Writer, mediatype string, b []byte) (int, error) {
-	mw := m.Get().Writer(mediatype, w)
+	mw := m.Writer(mediatype, w)
 	defer mw.Close()
 	return mw.Write(b)
 }
 
-func (m *Minifier) MinifyBytes(mediatype string, b []byte) ([]byte, error) {
-	return m.Get().Bytes(mediatype, b)
-}
-
-func (m *Minifier) MinifyString(mediatype string, s string) (string, error) {
-	return m.Get().String(mediatype, s)
-}
-
 func (m *Minifier) Minify(mediatype string, w io.Writer, r io.Reader) error {
 	return m.Get().Minify(mediatype, w, r)
 }
+
+func (m *Minifier) Writer(mediatype string, w io.Writer) io.WriteCloser {
+	return m.Get().Writer(mediatype, w)
+}
+
+func (m *Minifier) Reader(mediatype string, r io.Reader) io.Reader {
+	return m.Get().Reader(mediatype, r)
+}

render.go

@@ -40,10 +40,10 @@ func (a *goBlog) renderWithStatusCode(w http.ResponseWriter, r *http.Request, st
 	w.WriteHeader(statusCode)
 	// Render
 	buf := bufferpool.Get()
-	minWriter := a.min.Get().Writer(contenttype.HTML, buf)
+	mw := a.min.Writer(contenttype.HTML, buf)
-	hb := newHtmlBuilder(minWriter)
+	hb := newHtmlBuilder(mw)
 	f(hb, data)
-	_ = minWriter.Close()
+	_ = mw.Close()
 	_, _ = buf.WriteTo(w)
 	bufferpool.Put(buf)
 }

templateAssets.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"bytes"
+	"crypto/sha256"
+	"fmt"
 	"io"
 	"mime"
 	"net/http"
@@ -55,37 +57,26 @@ func (a *goBlog) initTemplateAssets() error {
 
 func (a *goBlog) compileAsset(name string, read io.Reader) (string, error) {
 	ext := path.Ext(name)
-	compiledExt := ext
-	var contentBuffer bytes.Buffer
 	switch ext {
 	case ".js":
-		if err := a.min.Minify(contenttype.JS, &contentBuffer, read); err != nil {
+		read = a.min.Reader(contenttype.JS, read)
-			return "", err
-		}
 	case ".css":
-		if err := a.min.Minify(contenttype.CSS, &contentBuffer, read); err != nil {
+		read = a.min.Reader(contenttype.CSS, read)
-			return "", err
-		}
 	case ".xml", ".xsl":
-		if err := a.min.Minify(contenttype.XML, &contentBuffer, read); err != nil {
+		read = a.min.Reader(contenttype.XML, read)
-			return "", err
-		}
-	default:
-		if _, err := io.Copy(&contentBuffer, read); err != nil {
-			return "", err
-		}
 	}
-	// Hashes
+	// Read file
-	hash, err := getSHA256(bytes.NewReader(contentBuffer.Bytes()))
+	hash := sha256.New()
+	body, err := io.ReadAll(io.TeeReader(read, hash))
 	if err != nil {
 		return "", err
 	}
 	// File name
-	compiledFileName := hash + compiledExt
+	compiledFileName := fmt.Sprintf("%x%s", hash.Sum(nil), ext)
 	// Create struct
 	a.assetFiles[compiledFileName] = &assetFile{
-		contentType: mime.TypeByExtension(compiledExt),
+		contentType: mime.TypeByExtension(ext),
-		body:        contentBuffer.Bytes(),
+		body:        body,
 	}
 	return compiledFileName, err
 }

tts.go

@@ -3,8 +3,10 @@ package main
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
 	"encoding/base64"
 	"errors"
+	"fmt"
 	"html"
 	"io"
 	"log"
@@ -105,18 +107,14 @@ func (a *goBlog) createPostTTSAudio(p *post) error {
 	}
 
 	// Merge partsBuffers into final buffer
-	var final bytes.Buffer
+	final := new(bytes.Buffer)
-	if err := mp3merge.MergeMP3(&final, partsBuffers...); err != nil {
+	hash := sha256.New()
+	if err := mp3merge.MergeMP3(io.MultiWriter(final, hash), partsBuffers...); err != nil {
 		return err
 	}
 
 	// Save audio
-	audioReader := bytes.NewReader(final.Bytes())
+	loc, err := a.saveMediaFile(fmt.Sprintf("%x.mp3", hash.Sum(nil)), final)
-	fileHash, err := getSHA256(audioReader)
-	if err != nil {
-		return err
-	}
-	loc, err := a.saveMediaFile(fileHash+".mp3", audioReader)
 	if err != nil {
 		return err
 	}

utils.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io"
@@ -217,21 +216,6 @@ func urlHasExt(rawUrl string, allowed ...string) (ext string, has bool) {
 	return ext, funk.ContainsString(allowed, strings.ToLower(ext))
 }
 
-// Get SHA-256 hash
-func getSHA256(file io.ReadSeeker) (hash string, err error) {
-	if _, err = file.Seek(0, 0); err != nil {
-		return "", err
-	}
-	h := sha256.New()
-	if _, err = io.Copy(h, file); err != nil {
-		return "", err
-	}
-	if _, err = file.Seek(0, 0); err != nil {
-		return "", err
-	}
-	return fmt.Sprintf("%x", h.Sum(nil)), nil
-}
-
 func mBytesString(size int64) string {
 	return fmt.Sprintf("%.2f MB", datasize.ByteSize(size).MBytes())
 }