Browse Source
* Implement encode middleware * Add missing break; and add missing JSON struct field tagmaster
committed by
GitHub
4 changed files with 347 additions and 0 deletions
@ -0,0 +1,286 @@ |
|||
// Package encode implements an encoder middleware for Caddy. The initial
|
|||
// enhancements related to Accept-Encoding, minimum content length, and
|
|||
// buffer/writer pools were adapted from https://github.com/xi2/httpgzip
|
|||
// then modified heavily to accommodate modular encoders. Code borrowed
|
|||
// from that repository is Copyright (c) 2015 The Httpgzip Authors.
|
|||
package encode |
|||
|
|||
import ( |
|||
"bytes" |
|||
"encoding/json" |
|||
"fmt" |
|||
"io" |
|||
"net/http" |
|||
"sort" |
|||
"strconv" |
|||
"strings" |
|||
"sync" |
|||
|
|||
"github.com/caddyserver/caddy2" |
|||
"github.com/caddyserver/caddy2/modules/caddyhttp" |
|||
) |
|||
|
|||
func init() { |
|||
caddy2.RegisterModule(caddy2.Module{ |
|||
Name: "http.middleware.encode", |
|||
New: func() interface{} { return new(Encode) }, |
|||
}) |
|||
} |
|||
|
|||
// Encode is a middleware which can encode responses.
|
|||
type Encode struct { |
|||
EncodingsRaw map[string]json.RawMessage `json:"encodings,omitempty"` |
|||
Prefer []string `json:"prefer,omitempty"` |
|||
MinLength int `json:"minimum_length,omitempty"` |
|||
|
|||
Encodings map[string]Encoding `json:"-"` |
|||
|
|||
writerPools map[string]*sync.Pool // TODO: these pools do not get reused through config reloads...
|
|||
} |
|||
|
|||
// Provision provisions enc.
|
|||
func (enc *Encode) Provision(ctx caddy2.Context) error { |
|||
enc.Encodings = make(map[string]Encoding) |
|||
enc.writerPools = make(map[string]*sync.Pool) |
|||
|
|||
for modName, rawMsg := range enc.EncodingsRaw { |
|||
val, err := ctx.LoadModule("http.encoders."+modName, rawMsg) |
|||
if err != nil { |
|||
return fmt.Errorf("loading encoder module '%s': %v", modName, err) |
|||
} |
|||
encoder := val.(Encoding) |
|||
enc.Encodings[modName] = encoder |
|||
|
|||
enc.writerPools[modName] = &sync.Pool{ |
|||
New: func() interface{} { |
|||
return encoder.NewEncoder() |
|||
}, |
|||
} |
|||
} |
|||
enc.EncodingsRaw = nil // allow GC to deallocate - TODO: Does this help?
|
|||
|
|||
if enc.MinLength == 0 { |
|||
enc.MinLength = defaultMinLength |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (enc *Encode) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { |
|||
for _, encName := range acceptedEncodings(r) { |
|||
if _, ok := enc.writerPools[encName]; !ok { |
|||
continue // encoding not offered
|
|||
} |
|||
w = enc.openResponseWriter(encName, w) |
|||
defer w.(*responseWriter).Close() |
|||
break |
|||
} |
|||
|
|||
return next.ServeHTTP(w, r) |
|||
} |
|||
|
|||
// openResponseWriter creates a new response writer that may (or may not)
|
|||
// encode the response with encodingName. The returned response writer MUST
|
|||
// be closed after the handler completes.
|
|||
func (enc *Encode) openResponseWriter(encodingName string, w http.ResponseWriter) *responseWriter { |
|||
buf := bufPool.Get().(*bytes.Buffer) |
|||
buf.Reset() |
|||
return &responseWriter{ |
|||
ResponseWriterWrapper: &caddyhttp.ResponseWriterWrapper{ResponseWriter: w}, |
|||
encodingName: encodingName, |
|||
buf: buf, |
|||
config: enc, |
|||
} |
|||
} |
|||
|
|||
// responseWriter writes to an underlying response writer
|
|||
// using the encoding represented by encodingName and
|
|||
// configured by config.
|
|||
type responseWriter struct { |
|||
*caddyhttp.ResponseWriterWrapper |
|||
encodingName string |
|||
w Encoder |
|||
buf *bytes.Buffer |
|||
config *Encode |
|||
statusCode int |
|||
} |
|||
|
|||
// WriteHeader stores the status to write when the time comes
|
|||
// to actually write the header.
|
|||
func (rw *responseWriter) WriteHeader(status int) { |
|||
rw.statusCode = status |
|||
} |
|||
|
|||
// Write writes to the response. If the response qualifies,
|
|||
// it is encoded using the encoder, which is initialized
|
|||
// if not done so already.
|
|||
func (rw *responseWriter) Write(p []byte) (int, error) { |
|||
var n, written int |
|||
var err error |
|||
|
|||
if rw.buf != nil && rw.config.MinLength > 0 { |
|||
written = rw.buf.Len() |
|||
_, err := rw.buf.Write(p) |
|||
if err != nil { |
|||
return 0, err |
|||
} |
|||
if rw.buf.Len() < rw.config.MinLength { |
|||
return len(p), nil |
|||
} |
|||
rw.init() |
|||
p = rw.buf.Bytes() |
|||
defer func() { |
|||
bufPool.Put(rw.buf) |
|||
rw.buf = nil |
|||
}() |
|||
} |
|||
|
|||
switch { |
|||
case rw.w != nil: |
|||
n, err = rw.w.Write(p) |
|||
default: |
|||
n, err = rw.ResponseWriter.Write(p) |
|||
} |
|||
n -= written |
|||
if n < 0 { |
|||
n = 0 |
|||
} |
|||
return n, err |
|||
} |
|||
|
|||
// init should be called once we know we are writing an encoded response.
|
|||
func (rw *responseWriter) init() { |
|||
if rw.Header().Get("Content-Encoding") == "" && rw.buf.Len() >= rw.config.MinLength { |
|||
rw.w = rw.config.writerPools[rw.encodingName].Get().(Encoder) |
|||
rw.w.Reset(rw.ResponseWriter) |
|||
rw.Header().Del("Content-Length") // https://github.com/golang/go/issues/14975
|
|||
rw.Header().Set("Content-Encoding", rw.encodingName) |
|||
} |
|||
rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content
|
|||
rw.ResponseWriter.WriteHeader(rw.statusCode) |
|||
} |
|||
|
|||
// Close writes any remaining buffered response and
|
|||
// deallocates any active resources.
|
|||
func (rw *responseWriter) Close() error { |
|||
var err error |
|||
if rw.buf != nil { |
|||
rw.init() |
|||
p := rw.buf.Bytes() |
|||
defer func() { |
|||
bufPool.Put(rw.buf) |
|||
rw.buf = nil |
|||
}() |
|||
switch { |
|||
case rw.w != nil: |
|||
_, err = rw.w.Write(p) |
|||
default: |
|||
_, err = rw.ResponseWriter.Write(p) |
|||
} |
|||
} |
|||
if rw.w != nil { |
|||
err2 := rw.w.Close() |
|||
if err2 != nil && err == nil { |
|||
err = err2 |
|||
} |
|||
rw.config.writerPools[rw.encodingName].Put(rw.w) |
|||
rw.w = nil |
|||
} |
|||
return err |
|||
} |
|||
|
|||
// acceptedEncodings returns the list of encodings that the
|
|||
// client supports, in descending order of preference. If
|
|||
// the Sec-WebSocket-Key header is present then non-identity
|
|||
// encodings are not considered. See
|
|||
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html.
|
|||
func acceptedEncodings(r *http.Request) []string { |
|||
acceptEncHeader := r.Header.Get("Accept-Encoding") |
|||
websocketKey := r.Header.Get("Sec-WebSocket-Key") |
|||
if acceptEncHeader == "" { |
|||
return []string{} |
|||
} |
|||
|
|||
var prefs []encodingPreference |
|||
|
|||
for _, accepted := range strings.Split(acceptEncHeader, ",") { |
|||
parts := strings.Split(accepted, ";") |
|||
encName := strings.ToLower(strings.TrimSpace(parts[0])) |
|||
|
|||
// determine q-factor
|
|||
qFactor := 1.0 |
|||
if len(parts) > 1 { |
|||
qFactorStr := strings.ToLower(strings.TrimSpace(parts[1])) |
|||
if strings.HasPrefix(qFactorStr, "q=") { |
|||
if qFactorFloat, err := strconv.ParseFloat(qFactorStr[2:], 32); err == nil { |
|||
if qFactorFloat >= 0 && qFactorFloat <= 1 { |
|||
qFactor = qFactorFloat |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
// encodings with q-factor of 0 are not accepted;
|
|||
// use a small theshold to account for float precision
|
|||
if qFactor < 0.00001 { |
|||
continue |
|||
} |
|||
|
|||
// don't encode WebSocket handshakes
|
|||
if websocketKey != "" && encName != "identity" { |
|||
continue |
|||
} |
|||
|
|||
prefs = append(prefs, encodingPreference{ |
|||
encoding: encName, |
|||
q: qFactor, |
|||
}) |
|||
} |
|||
|
|||
// sort preferences by descending q-factor
|
|||
sort.Slice(prefs, func(i, j int) bool { return prefs[i].q > prefs[j].q }) |
|||
|
|||
// TODO: If no preference, or same pref for all encodings,
|
|||
// and not websocket, use default encoding ordering (enc.Prefer)
|
|||
// for those which are accepted by the client
|
|||
|
|||
prefEncNames := make([]string, len(prefs)) |
|||
for i := range prefs { |
|||
prefEncNames[i] = prefs[i].encoding |
|||
} |
|||
|
|||
return prefEncNames |
|||
} |
|||
|
|||
// encodingPreference pairs an encoding with its q-factor.
|
|||
type encodingPreference struct { |
|||
encoding string |
|||
q float64 |
|||
} |
|||
|
|||
// Encoder is a type which can encode a stream of data.
|
|||
type Encoder interface { |
|||
io.WriteCloser |
|||
Reset(io.Writer) |
|||
} |
|||
|
|||
// Encoding is a type which can create encoders of its kind.
|
|||
type Encoding interface { |
|||
NewEncoder() Encoder |
|||
} |
|||
|
|||
var bufPool = sync.Pool{ |
|||
New: func() interface{} { |
|||
return new(bytes.Buffer) |
|||
}, |
|||
} |
|||
|
|||
// defaultMinLength is the minimum length at which to compress content.
|
|||
const defaultMinLength = 512 |
|||
|
|||
// Interface guards
|
|||
var ( |
|||
_ caddy2.Provisioner = (*Encode)(nil) |
|||
_ caddyhttp.MiddlewareHandler = (*Encode)(nil) |
|||
_ caddyhttp.HTTPInterfaces = (*responseWriter)(nil) |
|||
) |
@ -0,0 +1,36 @@ |
|||
package caddygzip |
|||
|
|||
import ( |
|||
"compress/flate" |
|||
"compress/gzip" // TODO: consider using https://github.com/klauspost/compress/gzip
|
|||
|
|||
"github.com/caddyserver/caddy2" |
|||
"github.com/caddyserver/caddy2/modules/caddyhttp/encode" |
|||
) |
|||
|
|||
func init() { |
|||
caddy2.RegisterModule(caddy2.Module{ |
|||
Name: "http.encoders.gzip", |
|||
New: func() interface{} { return new(Gzip) }, |
|||
}) |
|||
} |
|||
|
|||
// Gzip can create gzip encoders.
|
|||
type Gzip struct { |
|||
Level int `json:"level,omitempty"` |
|||
} |
|||
|
|||
// NewEncoder returns a new gzip writer.
|
|||
func (g Gzip) NewEncoder() encode.Encoder { |
|||
if g.Level <= flate.NoCompression { |
|||
g.Level = defaultGzipLevel |
|||
} |
|||
if g.Level > flate.BestCompression { |
|||
g.Level = flate.BestCompression |
|||
} |
|||
writer, _ := gzip.NewWriterLevel(nil, g.Level) |
|||
return writer |
|||
} |
|||
|
|||
// Informed from http://blog.klauspost.com/gzip-performance-for-go-webservers/
|
|||
var defaultGzipLevel = 5 |
@ -0,0 +1,23 @@ |
|||
package caddystd |
|||
|
|||
import ( |
|||
"github.com/caddyserver/caddy2" |
|||
"github.com/caddyserver/caddy2/modules/caddyhttp/encode" |
|||
"github.com/klauspost/compress/zstd" |
|||
) |
|||
|
|||
func init() { |
|||
caddy2.RegisterModule(caddy2.Module{ |
|||
Name: "http.encoders.zstd", |
|||
New: func() interface{} { return new(Zstd) }, |
|||
}) |
|||
} |
|||
|
|||
// Zstd can create zstd encoders.
|
|||
type Zstd struct{} |
|||
|
|||
// NewEncoder returns a new gzip writer.
|
|||
func (z Zstd) NewEncoder() encode.Encoder { |
|||
writer, _ := zstd.NewWriter(nil) |
|||
return writer |
|||
} |
Loading…
Reference in new issue