Skip to content

metrics: introduce custom registry #7051

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions context.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ type Context struct {
ancestry []Module
cleanupFuncs []func() // invoked at every config unload
exitFuncs []func(context.Context) // invoked at config unload ONLY IF the process is exiting (EXPERIMENTAL)
metricsRegistry *prometheus.Registry
metricsRegistry *registryGatherer
}

// NewContext provides a new context derived from the given
Expand All @@ -61,7 +61,8 @@ type Context struct {
// modules which are loaded will be properly unloaded.
// See standard library context package's documentation.
func NewContext(ctx Context) (Context, context.CancelFunc) {
newCtx := Context{moduleInstances: make(map[string][]Module), cfg: ctx.cfg, metricsRegistry: prometheus.NewPedanticRegistry()}
r := prometheus.NewPedanticRegistry()
newCtx := Context{moduleInstances: make(map[string][]Module), cfg: ctx.cfg, metricsRegistry: &registryGatherer{registry: r, gatherer: r}}
c, cancel := context.WithCancel(ctx.Context)
wrappedCancel := func() {
cancel()
Expand Down Expand Up @@ -103,7 +104,7 @@ func (ctx *Context) FileSystems() FileSystems {

// Returns the active metrics registry for the context
// EXPERIMENTAL: This API is subject to change.
func (ctx *Context) GetMetricsRegistry() *prometheus.Registry {
func (ctx *Context) GetMetricsRegistry() MetricsRegistererGatherer {
return ctx.metricsRegistry
}

Expand Down
63 changes: 63 additions & 0 deletions metrics.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
package caddy

import (
"errors"
"net/http"

"github.com/prometheus/client_golang/prometheus"
io_prometheus_client "github.com/prometheus/client_model/go"

"github.com/caddyserver/caddy/v2/internal/metrics"
)
Expand Down Expand Up @@ -82,3 +84,64 @@ func (d *delegator) WriteHeader(code int) {
func (d *delegator) Unwrap() http.ResponseWriter {
return d.ResponseWriter
}

type MetricsRegistererGatherer interface {
prometheus.Registerer
prometheus.Gatherer
}
type registryGatherer struct {
registry prometheus.Registerer
gatherer prometheus.Gatherer
}

// Gather implements prometheus.Gatherer.
func (r *registryGatherer) Gather() ([]*io_prometheus_client.MetricFamily, error) {
return r.gatherer.Gather()
}

// MustRegister calls `MustRegister` on the backing registry one collector
// at a time to capture the module at which the call may have panicked. Panics
// of duplicate registration are ignored.
func (r *registryGatherer) MustRegister(cs ...prometheus.Collector) {
var current prometheus.Collector
defer func() {
if r := recover(); r != nil {
err, ok := r.(error)
if !ok {
panic(r)
}
if !errors.Is(err, prometheus.AlreadyRegisteredError{
ExistingCollector: current,
NewCollector: current,
}) {
panic(err)
}
}
}()
for _, current = range cs {
r.registry.MustRegister(current)
}
}

// Register implements prometheus.Registerer. Errors of duplicate registration
// are ignored.
func (r *registryGatherer) Register(c prometheus.Collector) error {
if err := r.registry.Register(c); err != nil &&
!errors.Is(err, prometheus.AlreadyRegisteredError{
ExistingCollector: c,
NewCollector: c,
}) {
return err
}
return nil
}

// Unregister implements prometheus.Registerer.
func (r *registryGatherer) Unregister(c prometheus.Collector) bool {
return r.registry.Unregister(c)
}

var (
_ prometheus.Registerer = (*registryGatherer)(nil)
_ prometheus.Gatherer = (*registryGatherer)(nil)
)
15 changes: 2 additions & 13 deletions modules/caddyhttp/reverseproxy/metrics.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package reverseproxy

import (
"errors"
"runtime/debug"
"sync"
"time"
Expand All @@ -19,7 +18,7 @@ var reverseProxyMetrics = struct {
logger *zap.Logger
}{}

func initReverseProxyMetrics(handler *Handler, registry *prometheus.Registry) {
func initReverseProxyMetrics(handler *Handler, registry prometheus.Registerer) {
const ns, sub = "caddy", "reverse_proxy"

upstreamsLabels := []string{"upstream"}
Expand All @@ -32,17 +31,7 @@ func initReverseProxyMetrics(handler *Handler, registry *prometheus.Registry) {
}, upstreamsLabels)
})

// duplicate registration could happen if multiple sites with reverse proxy are configured; so ignore the error because
// there's no good way to capture having multiple sites with reverse proxy. If this happens, the metrics will be
// registered twice, but the second registration will be ignored.
if err := registry.Register(reverseProxyMetrics.upstreamsHealthy); err != nil &&
!errors.Is(err, prometheus.AlreadyRegisteredError{
ExistingCollector: reverseProxyMetrics.upstreamsHealthy,
NewCollector: reverseProxyMetrics.upstreamsHealthy,
}) {
panic(err)
}

registry.MustRegister(reverseProxyMetrics.upstreamsHealthy)
reverseProxyMetrics.logger = handler.logger.Named("reverse_proxy.metrics")
}

Expand Down
4 changes: 1 addition & 3 deletions modules/metrics/adminmetrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ import (
"errors"
"net/http"

"github.com/prometheus/client_golang/prometheus"

"github.com/caddyserver/caddy/v2"
)

Expand All @@ -33,7 +31,7 @@ func init() {
// See the Metrics module for a configurable endpoint that is usable if the
// Admin API is disabled.
type AdminMetrics struct {
registry *prometheus.Registry
registry caddy.MetricsRegistererGatherer

metricsHandler http.Handler
}
Expand Down
7 changes: 1 addition & 6 deletions modules/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,8 @@
package metrics

import (
"errors"
"net/http"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.uber.org/zap"

Expand Down Expand Up @@ -64,9 +62,6 @@ func (l *zapLogger) Println(v ...any) {
func (m *Metrics) Provision(ctx caddy.Context) error {
log := ctx.Logger()
registry := ctx.GetMetricsRegistry()
if registry == nil {
return errors.New("no metrics registry found")
}
m.metricsHandler = createMetricsHandler(&zapLogger{log}, !m.DisableOpenMetrics, registry)
return nil
}
Expand Down Expand Up @@ -112,7 +107,7 @@ var (
_ caddyfile.Unmarshaler = (*Metrics)(nil)
)

func createMetricsHandler(logger promhttp.Logger, enableOpenMetrics bool, registry *prometheus.Registry) http.Handler {
func createMetricsHandler(logger promhttp.Logger, enableOpenMetrics bool, registry caddy.MetricsRegistererGatherer) http.Handler {
return promhttp.InstrumentMetricHandler(registry,
promhttp.HandlerFor(registry, promhttp.HandlerOpts{
// will only log errors if logger is non-nil
Expand Down
Loading