Use custom compression middleware

This commit is contained in:
Jan-Lukas Else 2022-02-18 16:35:53 +01:00
parent 1f8c031e14
commit 6fc1e0dc03
9 changed files with 312 additions and 44 deletions

View File

@ -26,7 +26,7 @@ func (a *goBlog) serveEditor(w http.ResponseWriter, r *http.Request) {
func (a *goBlog) serveEditorPreview(w http.ResponseWriter, r *http.Request) { func (a *goBlog) serveEditorPreview(w http.ResponseWriter, r *http.Request) {
blog, _ := a.getBlog(r) blog, _ := a.getBlog(r)
c, err := ws.Accept(w, r, nil) c, err := ws.Accept(w, r, &ws.AcceptOptions{CompressionMode: ws.CompressionContextTakeover})
if err != nil { if err != nil {
return return
} }

2
go.mod
View File

@ -32,7 +32,7 @@ require (
github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4 github.com/jlelse/feeds v1.2.1-0.20210704161900-189f94254ad4
github.com/justinas/alice v1.2.0 github.com/justinas/alice v1.2.0
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9 github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9
github.com/klauspost/compress v1.14.2 github.com/klauspost/compress v1.14.3
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
github.com/lopezator/migrator v0.3.0 github.com/lopezator/migrator v0.3.0
github.com/mattn/go-sqlite3 v1.14.11 github.com/mattn/go-sqlite3 v1.14.11

4
go.sum
View File

@ -275,8 +275,8 @@ github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9 h1:+9REu9CK9D1AQ
github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9/go.mod h1:OvY5ZBrAC9kOvM2PZs9Lw0BH+5K7tjrT6T7SFhn27OA= github.com/kaorimatz/go-opml v0.0.0-20210201121027-bc8e2852d7f9/go.mod h1:OvY5ZBrAC9kOvM2PZs9Lw0BH+5K7tjrT6T7SFhn27OA=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw= github.com/klauspost/compress v1.14.3 h1:DQv1WP+iS4srNjibdnHtqu8JNWCDMluj5NzPnFJsnvk=
github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.14.3/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=

30
http.go
View File

@ -4,7 +4,6 @@ import (
"database/sql" "database/sql"
"errors" "errors"
"fmt" "fmt"
"io"
"log" "log"
"net" "net"
"net/http" "net/http"
@ -16,8 +15,7 @@ import (
"github.com/go-chi/chi/v5/middleware" "github.com/go-chi/chi/v5/middleware"
"github.com/justinas/alice" "github.com/justinas/alice"
"github.com/klauspost/compress/flate" "github.com/klauspost/compress/flate"
"github.com/klauspost/compress/gzip" "go.goblog.app/app/pkgs/httpcompress"
"go.goblog.app/app/pkgs/contenttype"
"go.goblog.app/app/pkgs/maprouter" "go.goblog.app/app/pkgs/maprouter"
"golang.org/x/net/context" "golang.org/x/net/context"
) )
@ -43,31 +41,7 @@ func (a *goBlog) startServer() (err error) {
if a.cfg.Server.Logging { if a.cfg.Server.Logging {
h = h.Append(a.logMiddleware) h = h.Append(a.logMiddleware)
} }
compressor := middleware.NewCompressor(flate.BestCompression, []string{ h = h.Append(middleware.Recoverer, httpcompress.Compress(flate.BestCompression), middleware.Heartbeat("/ping"))
contenttype.AS,
contenttype.ATOM,
contenttype.CSS,
contenttype.HTML,
contenttype.JS,
contenttype.JSON,
contenttype.JSONFeed,
contenttype.LDJSON,
contenttype.RSS,
contenttype.Text,
contenttype.XML,
"application/opensearchdescription+xml",
"application/jrd+json",
"application/xrd+xml",
}...)
compressor.SetEncoder("deflate", func(w io.Writer, level int) io.Writer {
cw, _ := flate.NewWriter(w, level)
return cw
})
compressor.SetEncoder("gzip", func(w io.Writer, level int) io.Writer {
cw, _ := gzip.NewWriterLevel(w, level)
return cw
})
h = h.Append(middleware.Recoverer, compressor.Handler, middleware.Heartbeat("/ping"))
if a.httpsConfigured(false) { if a.httpsConfigured(false) {
h = h.Append(a.securityHeaders) h = h.Append(a.securityHeaders)
} }

View File

@ -8,11 +8,11 @@ import (
) )
type fakeHttpClient struct { type fakeHttpClient struct {
*http.Client
req *http.Request
res *http.Response
handler http.Handler
mu sync.Mutex mu sync.Mutex
handler http.Handler
*http.Client
req *http.Request
res *http.Response
} }
func newFakeHttpClient() *fakeHttpClient { func newFakeHttpClient() *fakeHttpClient {

View File

@ -122,8 +122,8 @@ func (a *goBlog) renderMdTitle(s string) string {
// Links // Links
type customExtension struct { type customExtension struct {
absoluteLinks bool
publicAddress string publicAddress string
absoluteLinks bool
} }
func (l *customExtension) Extend(m goldmark.Markdown) { func (l *customExtension) Extend(m goldmark.Markdown) {
@ -136,8 +136,8 @@ func (l *customExtension) Extend(m goldmark.Markdown) {
} }
type customRenderer struct { type customRenderer struct {
absoluteLinks bool
publicAddress string publicAddress string
absoluteLinks bool
} }
func (c *customRenderer) RegisterFuncs(r renderer.NodeRendererFuncRegisterer) { func (c *customRenderer) RegisterFuncs(r renderer.NodeRendererFuncRegisterer) {

View File

@ -0,0 +1,293 @@
package httpcompress
import (
"bufio"
"errors"
"io"
"io/ioutil"
"net"
"net/http"
"strings"
"sync"
"github.com/klauspost/compress/flate"
"github.com/klauspost/compress/gzip"
"github.com/thoas/go-funk"
"go.goblog.app/app/pkgs/contenttype"
)
var defaultCompressibleContentTypes = []string{
contenttype.AS,
contenttype.ATOM,
contenttype.CSS,
contenttype.HTML,
contenttype.JS,
contenttype.JSON,
contenttype.JSONFeed,
contenttype.LDJSON,
contenttype.RSS,
contenttype.Text,
contenttype.XML,
"application/opensearchdescription+xml",
"application/jrd+json",
"application/xrd+xml",
}
// Compress is a middleware that compresses response
// body of a given content types to a data format based
// on Accept-Encoding request header. It uses a given
// compression level.
//
// Passing a compression level of 5 is sensible value
func Compress(level int, types ...string) func(next http.Handler) http.Handler {
return NewCompressor(level, types...).Handler
}
// Compressor represents a set of encoding configurations.
type Compressor struct {
// The mapping of pooled encoders to pools.
pooledEncoders map[string]*sync.Pool
// The set of content types allowed to be compressed.
allowedTypes map[string]interface{}
// The list of encoders in order of decreasing precedence.
encodingPrecedence []string
// The compression level.
level int
}
// NewCompressor creates a new Compressor that will handle encoding responses.
//
// The level should be one of the ones defined in the flate package.
// The types are the content types that are allowed to be compressed.
func NewCompressor(level int, types ...string) *Compressor {
// If types are provided, set those as the allowed types. If none are
// provided, use the default list.
allowedTypes := map[string]interface{}{}
for _, t := range funk.ShortIf(len(types) > 0, types, defaultCompressibleContentTypes).([]string) {
allowedTypes[t] = nil
}
c := &Compressor{
level: level,
pooledEncoders: map[string]*sync.Pool{},
allowedTypes: allowedTypes,
}
c.SetEncoder("deflate", encoderDeflate)
c.SetEncoder("gzip", encoderGzip)
return c
}
// SetEncoder can be used to set the implementation of a compression algorithm.
//
// The encoding should be a standardised identifier. See:
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding
func (c *Compressor) SetEncoder(encoding string, fn EncoderFunc) {
encoding = strings.ToLower(encoding)
if encoding == "" {
panic("the encoding can not be empty")
}
if fn == nil {
panic("attempted to set a nil encoder function")
}
// Deleted already registered encoder
delete(c.pooledEncoders, encoding)
c.pooledEncoders[encoding] = &sync.Pool{
New: func() interface{} {
return fn(ioutil.Discard, c.level)
},
}
for i, v := range c.encodingPrecedence {
if v == encoding {
c.encodingPrecedence = append(c.encodingPrecedence[:i], c.encodingPrecedence[i+1:]...)
break
}
}
c.encodingPrecedence = append([]string{encoding}, c.encodingPrecedence...)
}
// Handler returns a new middleware that will compress the response based on the
// current Compressor.
func (c *Compressor) Handler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
cw := &compressResponseWriter{
compressor: c,
ResponseWriter: w,
request: r,
}
next.ServeHTTP(cw, r)
cw.Close()
})
}
func matchAcceptEncoding(accepted []string, encoding string) bool {
return funk.ContainsString(accepted, encoding)
}
// An EncoderFunc is a function that wraps the provided io.Writer with a
// streaming compression algorithm and returns it.
//
// In case of failure, the function should return nil.
type EncoderFunc func(w io.Writer, level int) compressWriter
// Interface for types that allow resetting io.Writers.
type compressWriter interface {
io.Writer
Reset(w io.Writer)
Flush() error
}
type compressResponseWriter struct {
http.ResponseWriter // The response writer to delegate to.
encoder compressWriter // The encoder to use (if any).
cleanup func() // Cleanup function to reset and repool encoder.
compressor *Compressor // Holds the compressor configuration.
request *http.Request // The request that is being handled.
wroteHeader bool // Whether the header has been written.
closed bool // Whether the connection has been closed.
}
func (cw *compressResponseWriter) isCompressable() bool {
// Parse the first part of the Content-Type response header.
contentType := cw.Header().Get("Content-Type")
if idx := strings.Index(contentType, ";"); idx >= 0 {
contentType = contentType[0:idx]
}
// Is the content type compressable?
_, ok := cw.compressor.allowedTypes[contentType]
return ok
}
func (cw *compressResponseWriter) writer() io.Writer {
if cw.encoder != nil {
return cw.encoder
}
return cw.ResponseWriter
}
// selectEncoder returns the encoder, the name of the encoder, and a closer function.
func (cw *compressResponseWriter) selectEncoder() (compressWriter, string, func()) {
// Parse the names of all accepted algorithms from the header.
accepted := strings.Split(strings.ToLower(cw.request.Header.Get("Accept-Encoding")), ",")
// Find supported encoder by accepted list by precedence
for _, name := range cw.compressor.encodingPrecedence {
if matchAcceptEncoding(accepted, name) {
if pool, ok := cw.compressor.pooledEncoders[name]; ok {
encoder := pool.Get().(compressWriter)
cleanup := func() {
encoder.Reset(nil)
pool.Put(encoder)
}
encoder.Reset(cw.ResponseWriter)
return encoder, name, cleanup
}
}
}
// No encoder found to match the accepted encoding
return nil, "", nil
}
func (cw *compressResponseWriter) doCleanup() {
if cw.encoder != nil {
cw.encoder = nil
cw.cleanup()
cw.cleanup = nil
}
}
func (cw *compressResponseWriter) WriteHeader(code int) {
defer cw.ResponseWriter.WriteHeader(code)
if cw.wroteHeader {
return
}
cw.wroteHeader = true
if cw.Header().Get("Content-Encoding") != "" {
// Data has already been compressed.
return
}
if !cw.isCompressable() {
// Data is not compressable.
return
}
var encoding string
cw.encoder, encoding, cw.cleanup = cw.selectEncoder()
if encoding != "" {
cw.Header().Set("Content-Encoding", encoding)
cw.Header().Add("Vary", "Accept-Encoding")
// The content-length after compression is unknown
cw.Header().Del("Content-Length")
}
}
func (cw *compressResponseWriter) Write(p []byte) (int, error) {
if !cw.wroteHeader {
cw.WriteHeader(http.StatusOK)
}
return cw.writer().Write(p)
}
func (cw *compressResponseWriter) Flush() {
if cw.encoder != nil {
cw.encoder.Flush()
}
if f, ok := cw.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func (cw *compressResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
if hj, ok := cw.writer().(http.Hijacker); ok {
return hj.Hijack()
}
return nil, nil, errors.New("http.Hijacker is unavailable on the writer")
}
func (cw *compressResponseWriter) Push(target string, opts *http.PushOptions) error {
if ps, ok := cw.writer().(http.Pusher); ok {
return ps.Push(target, opts)
}
return errors.New("http.Pusher is unavailable on the writer")
}
func (cw *compressResponseWriter) Close() error {
if cw.closed {
return nil
}
cw.closed = true
defer cw.doCleanup()
if c, ok := cw.writer().(io.WriteCloser); ok {
return c.Close()
}
return errors.New("io.WriteCloser is unavailable on the writer")
}
func encoderGzip(w io.Writer, level int) compressWriter {
gw, err := gzip.NewWriterLevel(w, level)
if err != nil {
return nil
}
return gw
}
func encoderDeflate(w io.Writer, level int) compressWriter {
dw, err := flate.NewWriter(w, level)
if err != nil {
return nil
}
return dw
}

View File

@ -11,11 +11,12 @@ func (a *goBlog) isPrivate() bool {
} }
func (a *goBlog) privateModeHandler(next http.Handler) http.Handler { func (a *goBlog) privateModeHandler(next http.Handler) http.Handler {
private := alice.New(a.authMiddleware).Then(next)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if a.isPrivate() { if a.isPrivate() {
alice.New(a.authMiddleware).Then(next).ServeHTTP(w, r) private.ServeHTTP(w, r)
} else { return
next.ServeHTTP(w, r)
} }
next.ServeHTTP(w, r)
}) })
} }

View File

@ -9,10 +9,10 @@ import (
) )
type queueItem struct { type queueItem struct {
id int schedule time.Time
name string name string
content []byte content []byte
schedule *time.Time id int
} }
func (db *database) enqueue(name string, content []byte, schedule time.Time) error { func (db *database) enqueue(name string, content []byte, schedule time.Time) error {
@ -64,6 +64,6 @@ func (db *database) peekQueue(name string) (*queueItem, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
qi.schedule = &t qi.schedule = t
return qi, nil return qi, nil
} }