Dave Henderson
4 years ago
committed by
GitHub
12 changed files with 518 additions and 33 deletions
@ -0,0 +1,32 @@ |
|||
package caddy |
|||
|
|||
import ( |
|||
"github.com/prometheus/client_golang/prometheus" |
|||
"github.com/prometheus/client_golang/prometheus/promauto" |
|||
) |
|||
|
|||
// define and register the metrics used in this package.
|
|||
func init() { |
|||
prometheus.MustRegister(prometheus.NewBuildInfoCollector()) |
|||
|
|||
const ns, sub = "caddy", "admin" |
|||
|
|||
adminMetrics.requestCount = promauto.NewCounterVec(prometheus.CounterOpts{ |
|||
Namespace: ns, |
|||
Subsystem: sub, |
|||
Name: "http_requests_total", |
|||
Help: "Counter of requests made to the Admin API's HTTP endpoints.", |
|||
}, []string{"handler", "path", "code", "method"}) |
|||
adminMetrics.requestErrors = promauto.NewCounterVec(prometheus.CounterOpts{ |
|||
Namespace: ns, |
|||
Subsystem: sub, |
|||
Name: "http_request_errors_total", |
|||
Help: "Number of requests resulting in middleware errors.", |
|||
}, []string{"handler", "path", "method"}) |
|||
} |
|||
|
|||
// adminMetrics is a collection of metrics that can be tracked for the admin API.
|
|||
var adminMetrics = struct { |
|||
requestCount *prometheus.CounterVec |
|||
requestErrors *prometheus.CounterVec |
|||
}{} |
@ -0,0 +1,195 @@ |
|||
package caddyhttp |
|||
|
|||
import ( |
|||
"context" |
|||
"net/http" |
|||
"strconv" |
|||
"sync" |
|||
"time" |
|||
|
|||
"github.com/prometheus/client_golang/prometheus" |
|||
"github.com/prometheus/client_golang/prometheus/promauto" |
|||
) |
|||
|
|||
var httpMetrics = struct { |
|||
init sync.Once |
|||
requestInFlight *prometheus.GaugeVec |
|||
requestCount *prometheus.CounterVec |
|||
requestErrors *prometheus.CounterVec |
|||
requestDuration *prometheus.HistogramVec |
|||
requestSize *prometheus.HistogramVec |
|||
responseSize *prometheus.HistogramVec |
|||
responseDuration *prometheus.HistogramVec |
|||
}{ |
|||
init: sync.Once{}, |
|||
} |
|||
|
|||
func initHTTPMetrics() { |
|||
const ns, sub = "caddy", "http" |
|||
|
|||
basicLabels := []string{"server", "handler"} |
|||
httpMetrics.requestInFlight = promauto.NewGaugeVec(prometheus.GaugeOpts{ |
|||
Namespace: ns, |
|||
Subsystem: sub, |
|||
Name: "requests_in_flight", |
|||
Help: "Number of requests currently handled by this server.", |
|||
}, basicLabels) |
|||
httpMetrics.requestErrors = promauto.NewCounterVec(prometheus.CounterOpts{ |
|||
Namespace: ns, |
|||
Subsystem: sub, |
|||
Name: "request_errors_total", |
|||
Help: "Number of requests resulting in middleware errors.", |
|||
}, basicLabels) |
|||
httpMetrics.requestCount = promauto.NewCounterVec(prometheus.CounterOpts{ |
|||
Namespace: ns, |
|||
Subsystem: sub, |
|||
Name: "requests_total", |
|||
Help: "Counter of HTTP(S) requests made.", |
|||
}, basicLabels) |
|||
|
|||
// TODO: allow these to be customized in the config
|
|||
durationBuckets := prometheus.DefBuckets |
|||
sizeBuckets := prometheus.ExponentialBuckets(256, 4, 8) |
|||
|
|||
httpLabels := []string{"server", "handler", "code", "method"} |
|||
httpMetrics.requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ |
|||
Namespace: ns, |
|||
Subsystem: sub, |
|||
Name: "request_duration_seconds", |
|||
Help: "Histogram of round-trip request durations.", |
|||
Buckets: durationBuckets, |
|||
}, httpLabels) |
|||
httpMetrics.requestSize = promauto.NewHistogramVec(prometheus.HistogramOpts{ |
|||
Namespace: ns, |
|||
Subsystem: sub, |
|||
Name: "request_size_bytes", |
|||
Help: "Total size of the request. Includes body", |
|||
Buckets: sizeBuckets, |
|||
}, httpLabels) |
|||
httpMetrics.responseSize = promauto.NewHistogramVec(prometheus.HistogramOpts{ |
|||
Namespace: ns, |
|||
Subsystem: sub, |
|||
Name: "response_size_bytes", |
|||
Help: "Size of the returned response.", |
|||
Buckets: sizeBuckets, |
|||
}, httpLabels) |
|||
httpMetrics.responseDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ |
|||
Namespace: ns, |
|||
Subsystem: sub, |
|||
Name: "response_duration_seconds", |
|||
Help: "Histogram of times to first byte in response bodies.", |
|||
Buckets: durationBuckets, |
|||
}, httpLabels) |
|||
} |
|||
|
|||
type ctxKeyServerName struct{} |
|||
|
|||
// serverNameFromContext extracts the current server name from the context.
|
|||
// Returns "UNKNOWN" if none is available (should probably never happen?)
|
|||
func serverNameFromContext(ctx context.Context) string { |
|||
srvName, ok := ctx.Value(ctxKeyServerName{}).(string) |
|||
if !ok { |
|||
return "UNKNOWN" |
|||
} |
|||
return srvName |
|||
} |
|||
|
|||
func contextWithServerName(ctx context.Context, serverName string) context.Context { |
|||
return context.WithValue(ctx, ctxKeyServerName{}, serverName) |
|||
} |
|||
|
|||
type metricsInstrumentedHandler struct { |
|||
labels prometheus.Labels |
|||
statusLabels prometheus.Labels |
|||
mh MiddlewareHandler |
|||
} |
|||
|
|||
func newMetricsInstrumentedHandler(server, handler string, mh MiddlewareHandler) *metricsInstrumentedHandler { |
|||
httpMetrics.init.Do(func() { |
|||
initHTTPMetrics() |
|||
}) |
|||
|
|||
labels := prometheus.Labels{"server": server, "handler": handler} |
|||
statusLabels := prometheus.Labels{"server": server, "handler": handler, "code": "", "method": ""} |
|||
return &metricsInstrumentedHandler{labels, statusLabels, mh} |
|||
} |
|||
|
|||
func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handler) error { |
|||
inFlight := httpMetrics.requestInFlight.With(h.labels) |
|||
inFlight.Inc() |
|||
defer inFlight.Dec() |
|||
|
|||
statusLabels := prometheus.Labels{"method": r.Method} |
|||
for k, v := range h.labels { |
|||
statusLabels[k] = v |
|||
} |
|||
|
|||
start := time.Now() |
|||
|
|||
// This is a _bit_ of a hack - it depends on the ShouldBufferFunc always
|
|||
// being called when the headers are written.
|
|||
// Effectively the same behaviour as promhttp.InstrumentHandlerTimeToWriteHeader.
|
|||
writeHeaderRecorder := ShouldBufferFunc(func(status int, header http.Header) bool { |
|||
statusLabels["code"] = sanitizeCode(status) |
|||
ttfb := time.Since(start).Seconds() |
|||
observeWithExemplar(statusLabels, httpMetrics.responseDuration, ttfb) |
|||
return false |
|||
}) |
|||
wrec := NewResponseRecorder(w, nil, writeHeaderRecorder) |
|||
err := h.mh.ServeHTTP(wrec, r, next) |
|||
dur := time.Since(start).Seconds() |
|||
httpMetrics.requestCount.With(h.labels).Inc() |
|||
if err != nil { |
|||
httpMetrics.requestErrors.With(h.labels).Inc() |
|||
return err |
|||
} |
|||
|
|||
observeWithExemplar(statusLabels, httpMetrics.requestDuration, dur) |
|||
observeWithExemplar(statusLabels, httpMetrics.requestSize, float64(computeApproximateRequestSize(r))) |
|||
httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size())) |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func observeWithExemplar(l prometheus.Labels, o *prometheus.HistogramVec, value float64) { |
|||
obs := o.With(l) |
|||
if oe, ok := obs.(prometheus.ExemplarObserver); ok { |
|||
oe.ObserveWithExemplar(value, l) |
|||
return |
|||
} |
|||
// _should_ be a noop, but here just in case...
|
|||
obs.Observe(value) |
|||
} |
|||
|
|||
func sanitizeCode(code int) string { |
|||
if code == 0 { |
|||
return "200" |
|||
} |
|||
return strconv.Itoa(code) |
|||
|
|||
} |
|||
|
|||
// taken from https://github.com/prometheus/client_golang/blob/6007b2b5cae01203111de55f753e76d8dac1f529/prometheus/promhttp/instrument_server.go#L298
|
|||
func computeApproximateRequestSize(r *http.Request) int { |
|||
s := 0 |
|||
if r.URL != nil { |
|||
s += len(r.URL.String()) |
|||
} |
|||
|
|||
s += len(r.Method) |
|||
s += len(r.Proto) |
|||
for name, values := range r.Header { |
|||
s += len(name) |
|||
for _, value := range values { |
|||
s += len(value) |
|||
} |
|||
} |
|||
s += len(r.Host) |
|||
|
|||
// N.B. r.Form and r.MultipartForm are assumed to be included in r.URL.
|
|||
|
|||
if r.ContentLength != -1 { |
|||
s += int(r.ContentLength) |
|||
} |
|||
return s |
|||
} |
@ -0,0 +1,66 @@ |
|||
package caddyhttp |
|||
|
|||
import ( |
|||
"context" |
|||
"errors" |
|||
"net/http" |
|||
"net/http/httptest" |
|||
"testing" |
|||
|
|||
"github.com/prometheus/client_golang/prometheus/testutil" |
|||
) |
|||
|
|||
func TestServerNameFromContext(t *testing.T) { |
|||
ctx := context.Background() |
|||
expected := "UNKNOWN" |
|||
if actual := serverNameFromContext(ctx); actual != expected { |
|||
t.Errorf("Not equal: expected %q, but got %q", expected, actual) |
|||
} |
|||
|
|||
in := "foo" |
|||
ctx = contextWithServerName(ctx, in) |
|||
if actual := serverNameFromContext(ctx); actual != in { |
|||
t.Errorf("Not equal: expected %q, but got %q", in, actual) |
|||
} |
|||
} |
|||
|
|||
func TestMetricsInstrumentedHandler(t *testing.T) { |
|||
handlerErr := errors.New("oh noes") |
|||
response := []byte("hello world!") |
|||
h := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { |
|||
if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 1.0 { |
|||
t.Errorf("Not same: expected %#v, but got %#v", 1.0, actual) |
|||
} |
|||
if handlerErr == nil { |
|||
w.Write(response) |
|||
} |
|||
return handlerErr |
|||
}) |
|||
|
|||
mh := middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error { |
|||
return h.ServeHTTP(w, r) |
|||
}) |
|||
|
|||
ih := newMetricsInstrumentedHandler("foo", "bar", mh) |
|||
|
|||
r := httptest.NewRequest("GET", "/", nil) |
|||
w := httptest.NewRecorder() |
|||
|
|||
if actual := ih.ServeHTTP(w, r, h); actual != handlerErr { |
|||
t.Errorf("Not same: expected %#v, but got %#v", handlerErr, actual) |
|||
} |
|||
if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 0.0 { |
|||
t.Errorf("Not same: expected %#v, but got %#v", 0.0, actual) |
|||
} |
|||
|
|||
handlerErr = nil |
|||
if err := ih.ServeHTTP(w, r, h); err != nil { |
|||
t.Errorf("Received unexpected error: %w", err) |
|||
} |
|||
} |
|||
|
|||
type middlewareHandlerFunc func(http.ResponseWriter, *http.Request, Handler) error |
|||
|
|||
func (f middlewareHandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request, h Handler) error { |
|||
return f(w, r, h) |
|||
} |
@ -0,0 +1,55 @@ |
|||
// Copyright 2020 Matthew Holt and The Caddy Authors
|
|||
//
|
|||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|||
// you may not use this file except in compliance with the License.
|
|||
// You may obtain a copy of the License at
|
|||
//
|
|||
// http://www.apache.org/licenses/LICENSE-2.0
|
|||
//
|
|||
// Unless required by applicable law or agreed to in writing, software
|
|||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
// See the License for the specific language governing permissions and
|
|||
// limitations under the License.
|
|||
|
|||
package metrics |
|||
|
|||
import ( |
|||
"net/http" |
|||
|
|||
"github.com/caddyserver/caddy/v2" |
|||
) |
|||
|
|||
func init() { |
|||
caddy.RegisterModule(AdminMetrics{}) |
|||
} |
|||
|
|||
// AdminMetrics is a module that serves a metrics endpoint so that any gathered
|
|||
// metrics can be exposed for scraping. This module is not configurable, and
|
|||
// is permanently mounted to the admin API endpoint at "/metrics".
|
|||
// See the Metrics module for a configurable endpoint that is usable if the
|
|||
// Admin API is disabled.
|
|||
type AdminMetrics struct{} |
|||
|
|||
// CaddyModule returns the Caddy module information.
|
|||
func (AdminMetrics) CaddyModule() caddy.ModuleInfo { |
|||
return caddy.ModuleInfo{ |
|||
ID: "admin.api.metrics", |
|||
New: func() caddy.Module { return new(AdminMetrics) }, |
|||
} |
|||
} |
|||
|
|||
// Routes returns a route for the /metrics endpoint.
|
|||
func (m *AdminMetrics) Routes() []caddy.AdminRoute { |
|||
metricsHandler := createMetricsHandler(nil) |
|||
h := caddy.AdminHandlerFunc(func(w http.ResponseWriter, r *http.Request) error { |
|||
metricsHandler.ServeHTTP(w, r) |
|||
return nil |
|||
}) |
|||
return []caddy.AdminRoute{{Pattern: "/metrics", Handler: h}} |
|||
} |
|||
|
|||
// Interface guards
|
|||
var ( |
|||
_ caddy.AdminRouter = (*AdminMetrics)(nil) |
|||
) |
@ -0,0 +1,109 @@ |
|||
// Copyright 2020 Matthew Holt and The Caddy Authors
|
|||
//
|
|||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|||
// you may not use this file except in compliance with the License.
|
|||
// You may obtain a copy of the License at
|
|||
//
|
|||
// http://www.apache.org/licenses/LICENSE-2.0
|
|||
//
|
|||
// Unless required by applicable law or agreed to in writing, software
|
|||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
// See the License for the specific language governing permissions and
|
|||
// limitations under the License.
|
|||
|
|||
package metrics |
|||
|
|||
import ( |
|||
"net/http" |
|||
|
|||
"github.com/caddyserver/caddy/v2" |
|||
"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" |
|||
"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" |
|||
"github.com/caddyserver/caddy/v2/modules/caddyhttp" |
|||
|
|||
"go.uber.org/zap" |
|||
|
|||
"github.com/prometheus/client_golang/prometheus" |
|||
"github.com/prometheus/client_golang/prometheus/promhttp" |
|||
) |
|||
|
|||
func init() { |
|||
caddy.RegisterModule(Metrics{}) |
|||
httpcaddyfile.RegisterHandlerDirective("metrics", parseCaddyfile) |
|||
} |
|||
|
|||
// Metrics is a module that serves a /metrics endpoint so that any gathered
|
|||
// metrics can be exposed for scraping. This module is configurable by end-users
|
|||
// unlike AdminMetrics.
|
|||
type Metrics struct { |
|||
metricsHandler http.Handler |
|||
} |
|||
|
|||
// CaddyModule returns the Caddy module information.
|
|||
func (Metrics) CaddyModule() caddy.ModuleInfo { |
|||
return caddy.ModuleInfo{ |
|||
ID: "http.handlers.metrics", |
|||
New: func() caddy.Module { return new(Metrics) }, |
|||
} |
|||
} |
|||
|
|||
type zapLogger struct { |
|||
zl *zap.Logger |
|||
} |
|||
|
|||
func (l *zapLogger) Println(v ...interface{}) { |
|||
l.zl.Sugar().Error(v...) |
|||
} |
|||
|
|||
// Provision sets up m.
|
|||
func (m *Metrics) Provision(ctx caddy.Context) error { |
|||
log := ctx.Logger(m) |
|||
m.metricsHandler = createMetricsHandler(&zapLogger{log}) |
|||
return nil |
|||
} |
|||
|
|||
func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) { |
|||
var m Metrics |
|||
err := m.UnmarshalCaddyfile(h.Dispenser) |
|||
return m, err |
|||
} |
|||
|
|||
// UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax:
|
|||
//
|
|||
// metrics <matcher>
|
|||
//
|
|||
func (m *Metrics) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { |
|||
for d.Next() { |
|||
args := d.RemainingArgs() |
|||
if len(args) > 0 { |
|||
return d.ArgErr() |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (m Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { |
|||
m.metricsHandler.ServeHTTP(w, r) |
|||
return next.ServeHTTP(w, r) |
|||
} |
|||
|
|||
// Interface guards
|
|||
var ( |
|||
_ caddy.Provisioner = (*Metrics)(nil) |
|||
_ caddyhttp.MiddlewareHandler = (*Metrics)(nil) |
|||
_ caddyfile.Unmarshaler = (*Metrics)(nil) |
|||
) |
|||
|
|||
func createMetricsHandler(logger promhttp.Logger) http.Handler { |
|||
return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, |
|||
promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{ |
|||
// will only log errors if logger is non-nil
|
|||
ErrorLog: logger, |
|||
|
|||
// Allow OpenMetrics format to be negotiated - largely compatible,
|
|||
// except quantile/le label values always have a decimal.
|
|||
EnableOpenMetrics: true, |
|||
}), |
|||
) |
|||
} |
Loading…
Reference in new issue