Browse Source
* reverseproxy: Begin refactor to enable dynamic upstreams Streamed here: https://www.youtube.com/watch?v=hj7yzXb11jU * Implement SRV and A/AAA upstream sources Also get upstreams at every retry loop iteration instead of just once before the loop. See #4442. * Minor tweaks from review * Limit size of upstreams caches * Add doc notes deprecating LookupSRV * Provision dynamic upstreams Still WIP, preparing to preserve health checker functionality * Rejigger health checks Move active health check results into handler-specific Upstreams. Improve documentation regarding health checks and upstreams. * Deprecation notice * Add Caddyfile support, use `caddy.Duration` * Interface guards * Implement custom resolvers, add resolvers to http transport Caddyfile * SRV: fix Caddyfile `name` inline arg, remove proto condition * Use pointer receiver * Add debug logs Co-authored-by: Francis Lavoie <lavofr@gmail.com>master
Matt Holt
3 years ago
committed by
GitHub
10 changed files with 1056 additions and 281 deletions
@ -0,0 +1,116 @@ |
|||
:8884 { |
|||
reverse_proxy { |
|||
dynamic a foo 9000 |
|||
} |
|||
|
|||
reverse_proxy { |
|||
dynamic a { |
|||
name foo |
|||
port 9000 |
|||
refresh 5m |
|||
resolvers 8.8.8.8 8.8.4.4 |
|||
dial_timeout 2s |
|||
dial_fallback_delay 300ms |
|||
} |
|||
} |
|||
} |
|||
|
|||
:8885 { |
|||
reverse_proxy { |
|||
dynamic srv _api._tcp.example.com |
|||
} |
|||
|
|||
reverse_proxy { |
|||
dynamic srv { |
|||
service api |
|||
proto tcp |
|||
name example.com |
|||
refresh 5m |
|||
resolvers 8.8.8.8 8.8.4.4 |
|||
dial_timeout 1s |
|||
dial_fallback_delay -1s |
|||
} |
|||
} |
|||
} |
|||
|
|||
---------- |
|||
{ |
|||
"apps": { |
|||
"http": { |
|||
"servers": { |
|||
"srv0": { |
|||
"listen": [ |
|||
":8884" |
|||
], |
|||
"routes": [ |
|||
{ |
|||
"handle": [ |
|||
{ |
|||
"dynamic_upstreams": { |
|||
"name": "foo", |
|||
"port": "9000", |
|||
"source": "a" |
|||
}, |
|||
"handler": "reverse_proxy" |
|||
}, |
|||
{ |
|||
"dynamic_upstreams": { |
|||
"dial_fallback_delay": 300000000, |
|||
"dial_timeout": 2000000000, |
|||
"name": "foo", |
|||
"port": "9000", |
|||
"refresh": 300000000000, |
|||
"resolver": { |
|||
"addresses": [ |
|||
"8.8.8.8", |
|||
"8.8.4.4" |
|||
] |
|||
}, |
|||
"source": "a" |
|||
}, |
|||
"handler": "reverse_proxy" |
|||
} |
|||
] |
|||
} |
|||
] |
|||
}, |
|||
"srv1": { |
|||
"listen": [ |
|||
":8885" |
|||
], |
|||
"routes": [ |
|||
{ |
|||
"handle": [ |
|||
{ |
|||
"dynamic_upstreams": { |
|||
"name": "_api._tcp.example.com", |
|||
"source": "srv" |
|||
}, |
|||
"handler": "reverse_proxy" |
|||
}, |
|||
{ |
|||
"dynamic_upstreams": { |
|||
"dial_fallback_delay": -1000000000, |
|||
"dial_timeout": 1000000000, |
|||
"name": "example.com", |
|||
"proto": "tcp", |
|||
"refresh": 300000000000, |
|||
"resolver": { |
|||
"addresses": [ |
|||
"8.8.8.8", |
|||
"8.8.4.4" |
|||
] |
|||
}, |
|||
"service": "api", |
|||
"source": "srv" |
|||
}, |
|||
"handler": "reverse_proxy" |
|||
} |
|||
] |
|||
} |
|||
] |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
@ -0,0 +1,377 @@ |
|||
package reverseproxy |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
weakrand "math/rand" |
|||
"net" |
|||
"net/http" |
|||
"strconv" |
|||
"sync" |
|||
"time" |
|||
|
|||
"github.com/caddyserver/caddy/v2" |
|||
"go.uber.org/zap" |
|||
) |
|||
|
|||
func init() { |
|||
caddy.RegisterModule(SRVUpstreams{}) |
|||
caddy.RegisterModule(AUpstreams{}) |
|||
} |
|||
|
|||
// SRVUpstreams provides upstreams from SRV lookups.
|
|||
// The lookup DNS name can be configured either by
|
|||
// its individual parts (that is, specifying the
|
|||
// service, protocol, and name separately) to form
|
|||
// the standard "_service._proto.name" domain, or
|
|||
// the domain can be specified directly in name by
|
|||
// leaving service and proto empty. See RFC 2782.
|
|||
//
|
|||
// Lookups are cached and refreshed at the configured
|
|||
// refresh interval.
|
|||
//
|
|||
// Returned upstreams are sorted by priority and weight.
|
|||
type SRVUpstreams struct { |
|||
// The service label.
|
|||
Service string `json:"service,omitempty"` |
|||
|
|||
// The protocol label; either tcp or udp.
|
|||
Proto string `json:"proto,omitempty"` |
|||
|
|||
// The name label; or, if service and proto are
|
|||
// empty, the entire domain name to look up.
|
|||
Name string `json:"name,omitempty"` |
|||
|
|||
// The interval at which to refresh the SRV lookup.
|
|||
// Results are cached between lookups. Default: 1m
|
|||
Refresh caddy.Duration `json:"refresh,omitempty"` |
|||
|
|||
// Configures the DNS resolver used to resolve the
|
|||
// SRV address to SRV records.
|
|||
Resolver *UpstreamResolver `json:"resolver,omitempty"` |
|||
|
|||
// If Resolver is configured, how long to wait before
|
|||
// timing out trying to connect to the DNS server.
|
|||
DialTimeout caddy.Duration `json:"dial_timeout,omitempty"` |
|||
|
|||
// If Resolver is configured, how long to wait before
|
|||
// spawning an RFC 6555 Fast Fallback connection.
|
|||
// A negative value disables this.
|
|||
FallbackDelay caddy.Duration `json:"dial_fallback_delay,omitempty"` |
|||
|
|||
resolver *net.Resolver |
|||
|
|||
logger *zap.Logger |
|||
} |
|||
|
|||
// CaddyModule returns the Caddy module information.
|
|||
func (SRVUpstreams) CaddyModule() caddy.ModuleInfo { |
|||
return caddy.ModuleInfo{ |
|||
ID: "http.reverse_proxy.upstreams.srv", |
|||
New: func() caddy.Module { return new(SRVUpstreams) }, |
|||
} |
|||
} |
|||
|
|||
// String returns the RFC 2782 representation of the SRV domain.
|
|||
func (su SRVUpstreams) String() string { |
|||
return fmt.Sprintf("_%s._%s.%s", su.Service, su.Proto, su.Name) |
|||
} |
|||
|
|||
func (su *SRVUpstreams) Provision(ctx caddy.Context) error { |
|||
su.logger = ctx.Logger(su) |
|||
if su.Refresh == 0 { |
|||
su.Refresh = caddy.Duration(time.Minute) |
|||
} |
|||
|
|||
if su.Resolver != nil { |
|||
err := su.Resolver.ParseAddresses() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
d := &net.Dialer{ |
|||
Timeout: time.Duration(su.DialTimeout), |
|||
FallbackDelay: time.Duration(su.FallbackDelay), |
|||
} |
|||
su.resolver = &net.Resolver{ |
|||
PreferGo: true, |
|||
Dial: func(ctx context.Context, _, _ string) (net.Conn, error) { |
|||
//nolint:gosec
|
|||
addr := su.Resolver.netAddrs[weakrand.Intn(len(su.Resolver.netAddrs))] |
|||
return d.DialContext(ctx, addr.Network, addr.JoinHostPort(0)) |
|||
}, |
|||
} |
|||
} |
|||
if su.resolver == nil { |
|||
su.resolver = net.DefaultResolver |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) { |
|||
suStr := su.String() |
|||
|
|||
// first, use a cheap read-lock to return a cached result quickly
|
|||
srvsMu.RLock() |
|||
cached := srvs[suStr] |
|||
srvsMu.RUnlock() |
|||
if cached.isFresh() { |
|||
return cached.upstreams, nil |
|||
} |
|||
|
|||
// otherwise, obtain a write-lock to update the cached value
|
|||
srvsMu.Lock() |
|||
defer srvsMu.Unlock() |
|||
|
|||
// check to see if it's still stale, since we're now in a different
|
|||
// lock from when we first checked freshness; another goroutine might
|
|||
// have refreshed it in the meantime before we re-obtained our lock
|
|||
cached = srvs[suStr] |
|||
if cached.isFresh() { |
|||
return cached.upstreams, nil |
|||
} |
|||
|
|||
// prepare parameters and perform the SRV lookup
|
|||
repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer) |
|||
service := repl.ReplaceAll(su.Service, "") |
|||
proto := repl.ReplaceAll(su.Proto, "") |
|||
name := repl.ReplaceAll(su.Name, "") |
|||
|
|||
su.logger.Debug("refreshing SRV upstreams", |
|||
zap.String("service", service), |
|||
zap.String("proto", proto), |
|||
zap.String("name", name)) |
|||
|
|||
_, records, err := su.resolver.LookupSRV(r.Context(), service, proto, name) |
|||
if err != nil { |
|||
// From LookupSRV docs: "If the response contains invalid names, those records are filtered
|
|||
// out and an error will be returned alongside the the remaining results, if any." Thus, we
|
|||
// only return an error if no records were also returned.
|
|||
if len(records) == 0 { |
|||
return nil, err |
|||
} |
|||
su.logger.Warn("SRV records filtered", zap.Error(err)) |
|||
} |
|||
|
|||
upstreams := make([]*Upstream, len(records)) |
|||
for i, rec := range records { |
|||
su.logger.Debug("discovered SRV record", |
|||
zap.String("target", rec.Target), |
|||
zap.Uint16("port", rec.Port), |
|||
zap.Uint16("priority", rec.Priority), |
|||
zap.Uint16("weight", rec.Weight)) |
|||
addr := net.JoinHostPort(rec.Target, strconv.Itoa(int(rec.Port))) |
|||
upstreams[i] = &Upstream{Dial: addr} |
|||
} |
|||
|
|||
// before adding a new one to the cache (as opposed to replacing stale one), make room if cache is full
|
|||
if cached.freshness.IsZero() && len(srvs) >= 100 { |
|||
for randomKey := range srvs { |
|||
delete(srvs, randomKey) |
|||
break |
|||
} |
|||
} |
|||
|
|||
srvs[suStr] = srvLookup{ |
|||
srvUpstreams: su, |
|||
freshness: time.Now(), |
|||
upstreams: upstreams, |
|||
} |
|||
|
|||
return upstreams, nil |
|||
} |
|||
|
|||
type srvLookup struct { |
|||
srvUpstreams SRVUpstreams |
|||
freshness time.Time |
|||
upstreams []*Upstream |
|||
} |
|||
|
|||
func (sl srvLookup) isFresh() bool { |
|||
return time.Since(sl.freshness) < time.Duration(sl.srvUpstreams.Refresh) |
|||
} |
|||
|
|||
var ( |
|||
srvs = make(map[string]srvLookup) |
|||
srvsMu sync.RWMutex |
|||
) |
|||
|
|||
// AUpstreams provides upstreams from A/AAAA lookups.
|
|||
// Results are cached and refreshed at the configured
|
|||
// refresh interval.
|
|||
type AUpstreams struct { |
|||
// The domain name to look up.
|
|||
Name string `json:"name,omitempty"` |
|||
|
|||
// The port to use with the upstreams. Default: 80
|
|||
Port string `json:"port,omitempty"` |
|||
|
|||
// The interval at which to refresh the A lookup.
|
|||
// Results are cached between lookups. Default: 1m
|
|||
Refresh caddy.Duration `json:"refresh,omitempty"` |
|||
|
|||
// Configures the DNS resolver used to resolve the
|
|||
// domain name to A records.
|
|||
Resolver *UpstreamResolver `json:"resolver,omitempty"` |
|||
|
|||
// If Resolver is configured, how long to wait before
|
|||
// timing out trying to connect to the DNS server.
|
|||
DialTimeout caddy.Duration `json:"dial_timeout,omitempty"` |
|||
|
|||
// If Resolver is configured, how long to wait before
|
|||
// spawning an RFC 6555 Fast Fallback connection.
|
|||
// A negative value disables this.
|
|||
FallbackDelay caddy.Duration `json:"dial_fallback_delay,omitempty"` |
|||
|
|||
resolver *net.Resolver |
|||
} |
|||
|
|||
// CaddyModule returns the Caddy module information.
|
|||
func (AUpstreams) CaddyModule() caddy.ModuleInfo { |
|||
return caddy.ModuleInfo{ |
|||
ID: "http.reverse_proxy.upstreams.a", |
|||
New: func() caddy.Module { return new(AUpstreams) }, |
|||
} |
|||
} |
|||
|
|||
func (au AUpstreams) String() string { return au.Name } |
|||
|
|||
func (au *AUpstreams) Provision(_ caddy.Context) error { |
|||
if au.Refresh == 0 { |
|||
au.Refresh = caddy.Duration(time.Minute) |
|||
} |
|||
if au.Port == "" { |
|||
au.Port = "80" |
|||
} |
|||
|
|||
if au.Resolver != nil { |
|||
err := au.Resolver.ParseAddresses() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
d := &net.Dialer{ |
|||
Timeout: time.Duration(au.DialTimeout), |
|||
FallbackDelay: time.Duration(au.FallbackDelay), |
|||
} |
|||
au.resolver = &net.Resolver{ |
|||
PreferGo: true, |
|||
Dial: func(ctx context.Context, _, _ string) (net.Conn, error) { |
|||
//nolint:gosec
|
|||
addr := au.Resolver.netAddrs[weakrand.Intn(len(au.Resolver.netAddrs))] |
|||
return d.DialContext(ctx, addr.Network, addr.JoinHostPort(0)) |
|||
}, |
|||
} |
|||
} |
|||
if au.resolver == nil { |
|||
au.resolver = net.DefaultResolver |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (au AUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) { |
|||
auStr := au.String() |
|||
|
|||
// first, use a cheap read-lock to return a cached result quickly
|
|||
aAaaaMu.RLock() |
|||
cached := aAaaa[auStr] |
|||
aAaaaMu.RUnlock() |
|||
if cached.isFresh() { |
|||
return cached.upstreams, nil |
|||
} |
|||
|
|||
// otherwise, obtain a write-lock to update the cached value
|
|||
aAaaaMu.Lock() |
|||
defer aAaaaMu.Unlock() |
|||
|
|||
// check to see if it's still stale, since we're now in a different
|
|||
// lock from when we first checked freshness; another goroutine might
|
|||
// have refreshed it in the meantime before we re-obtained our lock
|
|||
cached = aAaaa[auStr] |
|||
if cached.isFresh() { |
|||
return cached.upstreams, nil |
|||
} |
|||
|
|||
repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer) |
|||
name := repl.ReplaceAll(au.Name, "") |
|||
port := repl.ReplaceAll(au.Port, "") |
|||
|
|||
ips, err := au.resolver.LookupIPAddr(r.Context(), name) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
upstreams := make([]*Upstream, len(ips)) |
|||
for i, ip := range ips { |
|||
upstreams[i] = &Upstream{ |
|||
Dial: net.JoinHostPort(ip.String(), port), |
|||
} |
|||
} |
|||
|
|||
// before adding a new one to the cache (as opposed to replacing stale one), make room if cache is full
|
|||
if cached.freshness.IsZero() && len(srvs) >= 100 { |
|||
for randomKey := range aAaaa { |
|||
delete(aAaaa, randomKey) |
|||
break |
|||
} |
|||
} |
|||
|
|||
aAaaa[auStr] = aLookup{ |
|||
aUpstreams: au, |
|||
freshness: time.Now(), |
|||
upstreams: upstreams, |
|||
} |
|||
|
|||
return upstreams, nil |
|||
} |
|||
|
|||
type aLookup struct { |
|||
aUpstreams AUpstreams |
|||
freshness time.Time |
|||
upstreams []*Upstream |
|||
} |
|||
|
|||
func (al aLookup) isFresh() bool { |
|||
return time.Since(al.freshness) < time.Duration(al.aUpstreams.Refresh) |
|||
} |
|||
|
|||
// UpstreamResolver holds the set of addresses of DNS resolvers of
|
|||
// upstream addresses
|
|||
type UpstreamResolver struct { |
|||
// The addresses of DNS resolvers to use when looking up the addresses of proxy upstreams.
|
|||
// It accepts [network addresses](/docs/conventions#network-addresses)
|
|||
// with port range of only 1. If the host is an IP address, it will be dialed directly to resolve the upstream server.
|
|||
// If the host is not an IP address, the addresses are resolved using the [name resolution convention](https://golang.org/pkg/net/#hdr-Name_Resolution) of the Go standard library.
|
|||
// If the array contains more than 1 resolver address, one is chosen at random.
|
|||
Addresses []string `json:"addresses,omitempty"` |
|||
netAddrs []caddy.NetworkAddress |
|||
} |
|||
|
|||
// ParseAddresses parses all the configured network addresses
|
|||
// and ensures they're ready to be used.
|
|||
func (u *UpstreamResolver) ParseAddresses() error { |
|||
for _, v := range u.Addresses { |
|||
addr, err := caddy.ParseNetworkAddress(v) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if addr.PortRangeSize() != 1 { |
|||
return fmt.Errorf("resolver address must have exactly one address; cannot call %v", addr) |
|||
} |
|||
u.netAddrs = append(u.netAddrs, addr) |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
var ( |
|||
aAaaa = make(map[string]aLookup) |
|||
aAaaaMu sync.RWMutex |
|||
) |
|||
|
|||
// Interface guards
|
|||
var ( |
|||
_ caddy.Provisioner = (*SRVUpstreams)(nil) |
|||
_ UpstreamSource = (*SRVUpstreams)(nil) |
|||
_ caddy.Provisioner = (*AUpstreams)(nil) |
|||
_ UpstreamSource = (*AUpstreams)(nil) |
|||
) |
Loading…
Reference in new issue