Merge pull request #936 from grafana/charleskorn/unregister-metrics

charleskorn · web-flow · commit 73cbc98c25ff · 2025-07-15T20:31:24.000+10:00
Unregister metrics emitted by `remote.WriteStorage` when closed
diff --git a/storage/remote/write.go b/storage/remote/write.go
@@ -268,6 +268,14 @@ func (rws *WriteStorage) Close() error {
 		q.Stop()
 	}
 	close(rws.quit)
+
+	rws.watcherMetrics.Unregister()
+	rws.liveReaderMetrics.Unregister()
+
+	if rws.reg != nil {
+		rws.reg.Unregister(rws.highestTimestamp.Gauge)
+	}
+
 	return nil
 }
 
diff --git a/tsdb/wlog/live_reader.go b/tsdb/wlog/live_reader.go
@@ -29,13 +29,15 @@ import (
 
 // LiveReaderMetrics holds all metrics exposed by the LiveReader.
 type LiveReaderMetrics struct {
+	reg                    prometheus.Registerer
 	readerCorruptionErrors *prometheus.CounterVec
 }
 
 // NewLiveReaderMetrics instantiates, registers and returns metrics to be injected
 // at LiveReader instantiation.
 func NewLiveReaderMetrics(reg prometheus.Registerer) *LiveReaderMetrics {
 	m := &LiveReaderMetrics{
+		reg: reg,
 		readerCorruptionErrors: prometheus.NewCounterVec(prometheus.CounterOpts{
 			Name: "prometheus_tsdb_wal_reader_corruption_errors_total",
 			Help: "Errors encountered when reading the WAL.",
@@ -49,6 +51,15 @@ func NewLiveReaderMetrics(reg prometheus.Registerer) *LiveReaderMetrics {
 	return m
 }
 
+// Unregister unregisters metrics emitted by this instance.
+func (m *LiveReaderMetrics) Unregister() {
+	if m.reg == nil {
+		return
+	}
+
+	m.reg.Unregister(m.readerCorruptionErrors)
+}
+
 // NewLiveReader returns a new live reader.
 func NewLiveReader(logger *slog.Logger, metrics *LiveReaderMetrics, r io.Reader) *LiveReader {
 	lr := &LiveReader{
diff --git a/tsdb/wlog/watcher.go b/tsdb/wlog/watcher.go
@@ -73,6 +73,7 @@ type WriteNotified interface {
 }
 
 type WatcherMetrics struct {
+	reg                   prometheus.Registerer
 	recordsRead           *prometheus.CounterVec
 	recordDecodeFails     *prometheus.CounterVec
 	samplesSentPreTailing *prometheus.CounterVec
@@ -113,6 +114,7 @@ type Watcher struct {
 
 func NewWatcherMetrics(reg prometheus.Registerer) *WatcherMetrics {
 	m := &WatcherMetrics{
+		reg: reg,
 		recordsRead: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
 				Namespace: "prometheus",
@@ -171,6 +173,19 @@ func NewWatcherMetrics(reg prometheus.Registerer) *WatcherMetrics {
 	return m
 }
 
+// Unregister unregisters metrics emitted by this instance.
+func (m *WatcherMetrics) Unregister() {
+	if m.reg == nil {
+		return
+	}
+
+	m.reg.Unregister(m.recordsRead)
+	m.reg.Unregister(m.recordDecodeFails)
+	m.reg.Unregister(m.samplesSentPreTailing)
+	m.reg.Unregister(m.currentSegment)
+	m.reg.Unregister(m.notificationsSkipped)
+}
+
 // NewWatcher creates a new WAL watcher for a given WriteTo.
 func NewWatcher(metrics *WatcherMetrics, readerMetrics *LiveReaderMetrics, logger *slog.Logger, name string, writer WriteTo, dir string, sendExemplars, sendHistograms, sendMetadata bool) *Watcher {
 	if logger == nil {

Original file line number	Diff line number	Diff line change
`@@ -268,6 +268,14 @@ func (rws *WriteStorage) Close() error {`
`268`	`268`	`q.Stop()`
`269`	`269`	`}`
`270`	`270`	`close(rws.quit)`
	`271`	`+`
	`272`	`+ rws.watcherMetrics.Unregister()`
	`273`	`+ rws.liveReaderMetrics.Unregister()`
	`274`	`+`
	`275`	`+ if rws.reg != nil {`
	`276`	`+ rws.reg.Unregister(rws.highestTimestamp.Gauge)`
	`277`	`+ }`
	`278`	`+`
`271`	`279`	`return nil`
`272`	`280`	`}`
`273`	`281`