Dave Henderson
4 years ago
committed by
GitHub
12 changed files with 518 additions and 33 deletions
@ -0,0 +1,32 @@ |
|||||
|
package caddy |
||||
|
|
||||
|
import ( |
||||
|
"github.com/prometheus/client_golang/prometheus" |
||||
|
"github.com/prometheus/client_golang/prometheus/promauto" |
||||
|
) |
||||
|
|
||||
|
// define and register the metrics used in this package.
|
||||
|
func init() { |
||||
|
prometheus.MustRegister(prometheus.NewBuildInfoCollector()) |
||||
|
|
||||
|
const ns, sub = "caddy", "admin" |
||||
|
|
||||
|
adminMetrics.requestCount = promauto.NewCounterVec(prometheus.CounterOpts{ |
||||
|
Namespace: ns, |
||||
|
Subsystem: sub, |
||||
|
Name: "http_requests_total", |
||||
|
Help: "Counter of requests made to the Admin API's HTTP endpoints.", |
||||
|
}, []string{"handler", "path", "code", "method"}) |
||||
|
adminMetrics.requestErrors = promauto.NewCounterVec(prometheus.CounterOpts{ |
||||
|
Namespace: ns, |
||||
|
Subsystem: sub, |
||||
|
Name: "http_request_errors_total", |
||||
|
Help: "Number of requests resulting in middleware errors.", |
||||
|
}, []string{"handler", "path", "method"}) |
||||
|
} |
||||
|
|
||||
|
// adminMetrics is a collection of metrics that can be tracked for the admin API.
|
||||
|
var adminMetrics = struct { |
||||
|
requestCount *prometheus.CounterVec |
||||
|
requestErrors *prometheus.CounterVec |
||||
|
}{} |
@ -0,0 +1,195 @@ |
|||||
|
package caddyhttp |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"net/http" |
||||
|
"strconv" |
||||
|
"sync" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/prometheus/client_golang/prometheus" |
||||
|
"github.com/prometheus/client_golang/prometheus/promauto" |
||||
|
) |
||||
|
|
||||
|
var httpMetrics = struct { |
||||
|
init sync.Once |
||||
|
requestInFlight *prometheus.GaugeVec |
||||
|
requestCount *prometheus.CounterVec |
||||
|
requestErrors *prometheus.CounterVec |
||||
|
requestDuration *prometheus.HistogramVec |
||||
|
requestSize *prometheus.HistogramVec |
||||
|
responseSize *prometheus.HistogramVec |
||||
|
responseDuration *prometheus.HistogramVec |
||||
|
}{ |
||||
|
init: sync.Once{}, |
||||
|
} |
||||
|
|
||||
|
func initHTTPMetrics() { |
||||
|
const ns, sub = "caddy", "http" |
||||
|
|
||||
|
basicLabels := []string{"server", "handler"} |
||||
|
httpMetrics.requestInFlight = promauto.NewGaugeVec(prometheus.GaugeOpts{ |
||||
|
Namespace: ns, |
||||
|
Subsystem: sub, |
||||
|
Name: "requests_in_flight", |
||||
|
Help: "Number of requests currently handled by this server.", |
||||
|
}, basicLabels) |
||||
|
httpMetrics.requestErrors = promauto.NewCounterVec(prometheus.CounterOpts{ |
||||
|
Namespace: ns, |
||||
|
Subsystem: sub, |
||||
|
Name: "request_errors_total", |
||||
|
Help: "Number of requests resulting in middleware errors.", |
||||
|
}, basicLabels) |
||||
|
httpMetrics.requestCount = promauto.NewCounterVec(prometheus.CounterOpts{ |
||||
|
Namespace: ns, |
||||
|
Subsystem: sub, |
||||
|
Name: "requests_total", |
||||
|
Help: "Counter of HTTP(S) requests made.", |
||||
|
}, basicLabels) |
||||
|
|
||||
|
// TODO: allow these to be customized in the config
|
||||
|
durationBuckets := prometheus.DefBuckets |
||||
|
sizeBuckets := prometheus.ExponentialBuckets(256, 4, 8) |
||||
|
|
||||
|
httpLabels := []string{"server", "handler", "code", "method"} |
||||
|
httpMetrics.requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ |
||||
|
Namespace: ns, |
||||
|
Subsystem: sub, |
||||
|
Name: "request_duration_seconds", |
||||
|
Help: "Histogram of round-trip request durations.", |
||||
|
Buckets: durationBuckets, |
||||
|
}, httpLabels) |
||||
|
httpMetrics.requestSize = promauto.NewHistogramVec(prometheus.HistogramOpts{ |
||||
|
Namespace: ns, |
||||
|
Subsystem: sub, |
||||
|
Name: "request_size_bytes", |
||||
|
Help: "Total size of the request. Includes body", |
||||
|
Buckets: sizeBuckets, |
||||
|
}, httpLabels) |
||||
|
httpMetrics.responseSize = promauto.NewHistogramVec(prometheus.HistogramOpts{ |
||||
|
Namespace: ns, |
||||
|
Subsystem: sub, |
||||
|
Name: "response_size_bytes", |
||||
|
Help: "Size of the returned response.", |
||||
|
Buckets: sizeBuckets, |
||||
|
}, httpLabels) |
||||
|
httpMetrics.responseDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ |
||||
|
Namespace: ns, |
||||
|
Subsystem: sub, |
||||
|
Name: "response_duration_seconds", |
||||
|
Help: "Histogram of times to first byte in response bodies.", |
||||
|
Buckets: durationBuckets, |
||||
|
}, httpLabels) |
||||
|
} |
||||
|
|
||||
|
type ctxKeyServerName struct{} |
||||
|
|
||||
|
// serverNameFromContext extracts the current server name from the context.
|
||||
|
// Returns "UNKNOWN" if none is available (should probably never happen?)
|
||||
|
func serverNameFromContext(ctx context.Context) string { |
||||
|
srvName, ok := ctx.Value(ctxKeyServerName{}).(string) |
||||
|
if !ok { |
||||
|
return "UNKNOWN" |
||||
|
} |
||||
|
return srvName |
||||
|
} |
||||
|
|
||||
|
func contextWithServerName(ctx context.Context, serverName string) context.Context { |
||||
|
return context.WithValue(ctx, ctxKeyServerName{}, serverName) |
||||
|
} |
||||
|
|
||||
|
type metricsInstrumentedHandler struct { |
||||
|
labels prometheus.Labels |
||||
|
statusLabels prometheus.Labels |
||||
|
mh MiddlewareHandler |
||||
|
} |
||||
|
|
||||
|
func newMetricsInstrumentedHandler(server, handler string, mh MiddlewareHandler) *metricsInstrumentedHandler { |
||||
|
httpMetrics.init.Do(func() { |
||||
|
initHTTPMetrics() |
||||
|
}) |
||||
|
|
||||
|
labels := prometheus.Labels{"server": server, "handler": handler} |
||||
|
statusLabels := prometheus.Labels{"server": server, "handler": handler, "code": "", "method": ""} |
||||
|
return &metricsInstrumentedHandler{labels, statusLabels, mh} |
||||
|
} |
||||
|
|
||||
|
func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handler) error { |
||||
|
inFlight := httpMetrics.requestInFlight.With(h.labels) |
||||
|
inFlight.Inc() |
||||
|
defer inFlight.Dec() |
||||
|
|
||||
|
statusLabels := prometheus.Labels{"method": r.Method} |
||||
|
for k, v := range h.labels { |
||||
|
statusLabels[k] = v |
||||
|
} |
||||
|
|
||||
|
start := time.Now() |
||||
|
|
||||
|
// This is a _bit_ of a hack - it depends on the ShouldBufferFunc always
|
||||
|
// being called when the headers are written.
|
||||
|
// Effectively the same behaviour as promhttp.InstrumentHandlerTimeToWriteHeader.
|
||||
|
writeHeaderRecorder := ShouldBufferFunc(func(status int, header http.Header) bool { |
||||
|
statusLabels["code"] = sanitizeCode(status) |
||||
|
ttfb := time.Since(start).Seconds() |
||||
|
observeWithExemplar(statusLabels, httpMetrics.responseDuration, ttfb) |
||||
|
return false |
||||
|
}) |
||||
|
wrec := NewResponseRecorder(w, nil, writeHeaderRecorder) |
||||
|
err := h.mh.ServeHTTP(wrec, r, next) |
||||
|
dur := time.Since(start).Seconds() |
||||
|
httpMetrics.requestCount.With(h.labels).Inc() |
||||
|
if err != nil { |
||||
|
httpMetrics.requestErrors.With(h.labels).Inc() |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
observeWithExemplar(statusLabels, httpMetrics.requestDuration, dur) |
||||
|
observeWithExemplar(statusLabels, httpMetrics.requestSize, float64(computeApproximateRequestSize(r))) |
||||
|
httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size())) |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func observeWithExemplar(l prometheus.Labels, o *prometheus.HistogramVec, value float64) { |
||||
|
obs := o.With(l) |
||||
|
if oe, ok := obs.(prometheus.ExemplarObserver); ok { |
||||
|
oe.ObserveWithExemplar(value, l) |
||||
|
return |
||||
|
} |
||||
|
// _should_ be a noop, but here just in case...
|
||||
|
obs.Observe(value) |
||||
|
} |
||||
|
|
||||
|
func sanitizeCode(code int) string { |
||||
|
if code == 0 { |
||||
|
return "200" |
||||
|
} |
||||
|
return strconv.Itoa(code) |
||||
|
|
||||
|
} |
||||
|
|
||||
|
// taken from https://github.com/prometheus/client_golang/blob/6007b2b5cae01203111de55f753e76d8dac1f529/prometheus/promhttp/instrument_server.go#L298
|
||||
|
func computeApproximateRequestSize(r *http.Request) int { |
||||
|
s := 0 |
||||
|
if r.URL != nil { |
||||
|
s += len(r.URL.String()) |
||||
|
} |
||||
|
|
||||
|
s += len(r.Method) |
||||
|
s += len(r.Proto) |
||||
|
for name, values := range r.Header { |
||||
|
s += len(name) |
||||
|
for _, value := range values { |
||||
|
s += len(value) |
||||
|
} |
||||
|
} |
||||
|
s += len(r.Host) |
||||
|
|
||||
|
// N.B. r.Form and r.MultipartForm are assumed to be included in r.URL.
|
||||
|
|
||||
|
if r.ContentLength != -1 { |
||||
|
s += int(r.ContentLength) |
||||
|
} |
||||
|
return s |
||||
|
} |
@ -0,0 +1,66 @@ |
|||||
|
package caddyhttp |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"errors" |
||||
|
"net/http" |
||||
|
"net/http/httptest" |
||||
|
"testing" |
||||
|
|
||||
|
"github.com/prometheus/client_golang/prometheus/testutil" |
||||
|
) |
||||
|
|
||||
|
func TestServerNameFromContext(t *testing.T) { |
||||
|
ctx := context.Background() |
||||
|
expected := "UNKNOWN" |
||||
|
if actual := serverNameFromContext(ctx); actual != expected { |
||||
|
t.Errorf("Not equal: expected %q, but got %q", expected, actual) |
||||
|
} |
||||
|
|
||||
|
in := "foo" |
||||
|
ctx = contextWithServerName(ctx, in) |
||||
|
if actual := serverNameFromContext(ctx); actual != in { |
||||
|
t.Errorf("Not equal: expected %q, but got %q", in, actual) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestMetricsInstrumentedHandler(t *testing.T) { |
||||
|
handlerErr := errors.New("oh noes") |
||||
|
response := []byte("hello world!") |
||||
|
h := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { |
||||
|
if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 1.0 { |
||||
|
t.Errorf("Not same: expected %#v, but got %#v", 1.0, actual) |
||||
|
} |
||||
|
if handlerErr == nil { |
||||
|
w.Write(response) |
||||
|
} |
||||
|
return handlerErr |
||||
|
}) |
||||
|
|
||||
|
mh := middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error { |
||||
|
return h.ServeHTTP(w, r) |
||||
|
}) |
||||
|
|
||||
|
ih := newMetricsInstrumentedHandler("foo", "bar", mh) |
||||
|
|
||||
|
r := httptest.NewRequest("GET", "/", nil) |
||||
|
w := httptest.NewRecorder() |
||||
|
|
||||
|
if actual := ih.ServeHTTP(w, r, h); actual != handlerErr { |
||||
|
t.Errorf("Not same: expected %#v, but got %#v", handlerErr, actual) |
||||
|
} |
||||
|
if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 0.0 { |
||||
|
t.Errorf("Not same: expected %#v, but got %#v", 0.0, actual) |
||||
|
} |
||||
|
|
||||
|
handlerErr = nil |
||||
|
if err := ih.ServeHTTP(w, r, h); err != nil { |
||||
|
t.Errorf("Received unexpected error: %w", err) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
type middlewareHandlerFunc func(http.ResponseWriter, *http.Request, Handler) error |
||||
|
|
||||
|
func (f middlewareHandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request, h Handler) error { |
||||
|
return f(w, r, h) |
||||
|
} |
@ -0,0 +1,55 @@ |
|||||
|
// Copyright 2020 Matthew Holt and The Caddy Authors
|
||||
|
//
|
||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
// you may not use this file except in compliance with the License.
|
||||
|
// You may obtain a copy of the License at
|
||||
|
//
|
||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
//
|
||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
|
// See the License for the specific language governing permissions and
|
||||
|
// limitations under the License.
|
||||
|
|
||||
|
package metrics |
||||
|
|
||||
|
import ( |
||||
|
"net/http" |
||||
|
|
||||
|
"github.com/caddyserver/caddy/v2" |
||||
|
) |
||||
|
|
||||
|
func init() { |
||||
|
caddy.RegisterModule(AdminMetrics{}) |
||||
|
} |
||||
|
|
||||
|
// AdminMetrics is a module that serves a metrics endpoint so that any gathered
|
||||
|
// metrics can be exposed for scraping. This module is not configurable, and
|
||||
|
// is permanently mounted to the admin API endpoint at "/metrics".
|
||||
|
// See the Metrics module for a configurable endpoint that is usable if the
|
||||
|
// Admin API is disabled.
|
||||
|
type AdminMetrics struct{} |
||||
|
|
||||
|
// CaddyModule returns the Caddy module information.
|
||||
|
func (AdminMetrics) CaddyModule() caddy.ModuleInfo { |
||||
|
return caddy.ModuleInfo{ |
||||
|
ID: "admin.api.metrics", |
||||
|
New: func() caddy.Module { return new(AdminMetrics) }, |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Routes returns a route for the /metrics endpoint.
|
||||
|
func (m *AdminMetrics) Routes() []caddy.AdminRoute { |
||||
|
metricsHandler := createMetricsHandler(nil) |
||||
|
h := caddy.AdminHandlerFunc(func(w http.ResponseWriter, r *http.Request) error { |
||||
|
metricsHandler.ServeHTTP(w, r) |
||||
|
return nil |
||||
|
}) |
||||
|
return []caddy.AdminRoute{{Pattern: "/metrics", Handler: h}} |
||||
|
} |
||||
|
|
||||
|
// Interface guards
|
||||
|
var ( |
||||
|
_ caddy.AdminRouter = (*AdminMetrics)(nil) |
||||
|
) |
@ -0,0 +1,109 @@ |
|||||
|
// Copyright 2020 Matthew Holt and The Caddy Authors
|
||||
|
//
|
||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
// you may not use this file except in compliance with the License.
|
||||
|
// You may obtain a copy of the License at
|
||||
|
//
|
||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
//
|
||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
|
// See the License for the specific language governing permissions and
|
||||
|
// limitations under the License.
|
||||
|
|
||||
|
package metrics |
||||
|
|
||||
|
import ( |
||||
|
"net/http" |
||||
|
|
||||
|
"github.com/caddyserver/caddy/v2" |
||||
|
"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" |
||||
|
"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" |
||||
|
"github.com/caddyserver/caddy/v2/modules/caddyhttp" |
||||
|
|
||||
|
"go.uber.org/zap" |
||||
|
|
||||
|
"github.com/prometheus/client_golang/prometheus" |
||||
|
"github.com/prometheus/client_golang/prometheus/promhttp" |
||||
|
) |
||||
|
|
||||
|
func init() { |
||||
|
caddy.RegisterModule(Metrics{}) |
||||
|
httpcaddyfile.RegisterHandlerDirective("metrics", parseCaddyfile) |
||||
|
} |
||||
|
|
||||
|
// Metrics is a module that serves a /metrics endpoint so that any gathered
|
||||
|
// metrics can be exposed for scraping. This module is configurable by end-users
|
||||
|
// unlike AdminMetrics.
|
||||
|
type Metrics struct { |
||||
|
metricsHandler http.Handler |
||||
|
} |
||||
|
|
||||
|
// CaddyModule returns the Caddy module information.
|
||||
|
func (Metrics) CaddyModule() caddy.ModuleInfo { |
||||
|
return caddy.ModuleInfo{ |
||||
|
ID: "http.handlers.metrics", |
||||
|
New: func() caddy.Module { return new(Metrics) }, |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
type zapLogger struct { |
||||
|
zl *zap.Logger |
||||
|
} |
||||
|
|
||||
|
func (l *zapLogger) Println(v ...interface{}) { |
||||
|
l.zl.Sugar().Error(v...) |
||||
|
} |
||||
|
|
||||
|
// Provision sets up m.
|
||||
|
func (m *Metrics) Provision(ctx caddy.Context) error { |
||||
|
log := ctx.Logger(m) |
||||
|
m.metricsHandler = createMetricsHandler(&zapLogger{log}) |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) { |
||||
|
var m Metrics |
||||
|
err := m.UnmarshalCaddyfile(h.Dispenser) |
||||
|
return m, err |
||||
|
} |
||||
|
|
||||
|
// UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax:
|
||||
|
//
|
||||
|
// metrics <matcher>
|
||||
|
//
|
||||
|
func (m *Metrics) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { |
||||
|
for d.Next() { |
||||
|
args := d.RemainingArgs() |
||||
|
if len(args) > 0 { |
||||
|
return d.ArgErr() |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (m Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { |
||||
|
m.metricsHandler.ServeHTTP(w, r) |
||||
|
return next.ServeHTTP(w, r) |
||||
|
} |
||||
|
|
||||
|
// Interface guards
|
||||
|
var ( |
||||
|
_ caddy.Provisioner = (*Metrics)(nil) |
||||
|
_ caddyhttp.MiddlewareHandler = (*Metrics)(nil) |
||||
|
_ caddyfile.Unmarshaler = (*Metrics)(nil) |
||||
|
) |
||||
|
|
||||
|
func createMetricsHandler(logger promhttp.Logger) http.Handler { |
||||
|
return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, |
||||
|
promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{ |
||||
|
// will only log errors if logger is non-nil
|
||||
|
ErrorLog: logger, |
||||
|
|
||||
|
// Allow OpenMetrics format to be negotiated - largely compatible,
|
||||
|
// except quantile/le label values always have a decimal.
|
||||
|
EnableOpenMetrics: true, |
||||
|
}), |
||||
|
) |
||||
|
} |
Loading…
Reference in new issue