spanner: leak of grpc-go newClientStreamWithParams #12659
Description
Client
Spanner
Code and Dependencies
After bumping to spanner@v1.84.0 in our QA environment we have a lot of goroutine leaks at newClientStreamWithParams.
pprof trace for the
goroutine profile: total 114375
112067 @ 0x47ab4e 0x457b77 0xf2bf4c 0x483021
# labels: {"name":"accounting:rollup", "subsystem":"core"}
# 0xf2bf4b google.golang.org/grpc.newClientStreamWithParams.func4+0x8b /go/pkg/mod/google.golang.org/grpc@v1.74.2/stream.go:398
1963 @ 0x47ab4e 0x457b77 0xf2bf4c 0x483021
# labels: {"name":"accounting:rollup-archive", "subsystem":"core"}
# 0xf2bf4b google.golang.org/grpc.newClientStreamWithParams.func4+0x8b /go/pkg/mod/google.golang.org/grpc@v1.74.2/stream.go:398
...
Where
goroutine 1750 [select, 36 minutes]:
google.golang.org/grpc.newClientStreamWithParams.func4()
/go/pkg/mod/google.golang.org/grpc@v1.74.2/stream.go:398 +0x8c
created by google.golang.org/grpc.newClientStreamWithParams in goroutine 715
/go/pkg/mod/google.golang.org/grpc@v1.74.2/stream.go:397 +0xe09
goroutine 715 [select]:
google.golang.org/grpc/internal/transport.(*ClientStream).waitOnHeader(0xc0692554a0)
/go/pkg/mod/google.golang.org/grpc@v1.74.2/internal/transport/client_stream.go:92 +0x65
google.golang.org/grpc/internal/transport.(*ClientStream).Header(0xc0692554a0)
/go/pkg/mod/google.golang.org/grpc@v1.74.2/internal/transport/client_stream.go:122 +0x27
google.golang.org/grpc.(*clientStream).Header.func1(0x1?)
/go/pkg/mod/google.golang.org/grpc@v1.74.2/stream.go:821 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc032306120, 0xc028c05f68, 0xc028c05f58)
/go/pkg/mod/google.golang.org/grpc@v1.74.2/stream.go:796 +0x13b
google.golang.org/grpc.(*clientStream).Header(0xc032306120)
/go/pkg/mod/google.golang.org/grpc@v1.74.2/stream.go:819 +0x56
cloud.google.com/go/spanner.(*grpcSpannerClient).ExecuteStreamingSql(0xc001224810, {0x3c317b8, 0xc02b9cb290}, 0xc002045ec0, {0xc0727a8ba0, 0x1, 0x1})
Full stack trace
goroutine 1750 [select, 36 minutes]:
google.golang.org/grpc.newClientStreamWithParams.func4()
/go/pkg/mod/google.golang.org/grpc@v1.74.2/stream.go:398 +0x8c
created by google.golang.org/grpc.newClientStreamWithParams in goroutine 715
/go/pkg/mod/google.golang.org/grpc@v1.74.2/stream.go:397 +0xe09
goroutine 715 [select]:
google.golang.org/grpc/internal/transport.(*ClientStream).waitOnHeader(0xc0692554a0)
/go/pkg/mod/google.golang.org/grpc@v1.74.2/internal/transport/client_stream.go:92 +0x65
google.golang.org/grpc/internal/transport.(*ClientStream).Header(0xc0692554a0)
/go/pkg/mod/google.golang.org/grpc@v1.74.2/internal/transport/client_stream.go:122 +0x27
google.golang.org/grpc.(*clientStream).Header.func1(0x1?)
/go/pkg/mod/google.golang.org/grpc@v1.74.2/stream.go:821 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc032306120, 0xc028c05f68, 0xc028c05f58)
/go/pkg/mod/google.golang.org/grpc@v1.74.2/stream.go:796 +0x13b
google.golang.org/grpc.(*clientStream).Header(0xc032306120)
/go/pkg/mod/google.golang.org/grpc@v1.74.2/stream.go:819 +0x56
cloud.google.com/go/spanner.(*grpcSpannerClient).ExecuteStreamingSql(0xc001224810, {0x3c317b8, 0xc02b9cb290}, 0xc002045ec0, {0xc0727a8ba0, 0x1, 0x1})
/go/pkg/mod/cloud.google.com/go/spanner@v1.84.0/grpc_client.go:257 +0x18b
cloud.google.com/go/spanner.(*txReadOnly).query.func2({0x3c317b8, 0xc02b9cb290}, {0x0?, 0x3bf0520?, 0x2e1b3e0?}, {0xc0727a8ba0, 0x1, 0x1})
/go/pkg/mod/cloud.google.com/go/spanner@v1.84.0/transaction.go:709 +0x170
cloud.google.com/go/spanner.(*resumableStreamDecoder).next(0xc0020843c0, 0xc05433ed00)
/go/pkg/mod/cloud.google.com/go/spanner@v1.84.0/read.go:581 +0x396
cloud.google.com/go/spanner.(*RowIterator).Next(0xc03e15abb0)
/go/pkg/mod/cloud.google.com/go/spanner@v1.84.0/read.go:185 +0x225
github.com/googleapis/go-sql-spanner.execTransactionalDML({0x3c30ef8, 0xc07243d540}, {0x3bfd1a0, 0xc000cdc540}, {{0xc049751400?, 0x47d009?}, 0xc028fe8e70?}, 0xc03ae19a50, {0x0, 0x0, ...})
/go/pkg/mod/github.com/googleapis/go-sql-spanner@v1.11.1-0.20250214171559-1bccea5dfec5/conn.go:1106 +0x14f
github.com/googleapis/go-sql-spanner.(*readWriteTransaction).ExecContext.func1({0x3c30ef8?, 0xc07243d540?})
/go/pkg/mod/github.com/googleapis/go-sql-spanner@v1.11.1-0.20250214171559-1bccea5dfec5/transaction.go:455 +0x89
github.com/googleapis/go-sql-spanner.(*readWriteTransaction).runWithRetry(0xc0736cce40, {0x3c30ef8, 0xc07243d540}, 0xc028c06668)
/go/pkg/mod/github.com/googleapis/go-sql-spanner@v1.11.1-0.20250214171559-1bccea5dfec5/transaction.go:310 +0x75
github.com/googleapis/go-sql-spanner.(*readWriteTransaction).ExecContext(0xc0736cce40, {0x3c30ef8, 0xc07243d540}, {{0xc049751400?, 0x0?}, 0xc028fe8e70?}, 0xc03ae19a50, {0x0, 0x0, 0x0, ...})
/go/pkg/mod/github.com/googleapis/go-sql-spanner@v1.11.1-0.20250214171559-1bccea5dfec5/transaction.go:454 +0x228
github.com/googleapis/go-sql-spanner.(*conn).execContext(0xc001dd5680, {0x3c30ef8, 0xc07243d540}, {0x33169ff, 0xfe}, {0x0, 0x0, {{0x0, 0x0}, {0x0, ...}, ...}, ...}, ...)
/go/pkg/mod/github.com/googleapis/go-sql-spanner@v1.11.1-0.20250214171559-1bccea5dfec5/conn.go:831 +0x5cb
github.com/googleapis/go-sql-spanner.(*conn).ExecContext(0xc001dd5680, {0x3c30ef8, 0xc07243d540}, {0x33169ff, 0xfe}, {0xc001b45680, 0x9, 0x9})
/go/pkg/mod/github.com/googleapis/go-sql-spanner@v1.11.1-0.20250214171559-1bccea5dfec5/conn.go:779 +0x13c
database/sql.ctxDriverExec({0x3c30ef8?, 0xc07243d540?}, {0x7b9122e0e140?, 0xc001dd5680?}, {0x0?, 0x0?}, {0x33169ff?, 0x5364198?}, {0xc001b45680, 0x9, ...})
/usr/local/go/src/database/sql/ctxutil.go:31 +0xd7
database/sql.(*DB).execDC.func2()
/usr/local/go/src/database/sql/sql.go:1713 +0x159
database/sql.withLock({0x3c1d150, 0xc063893480}, 0xc028c06d48)
/usr/local/go/src/database/sql/sql.go:3574 +0x71
database/sql.(*DB).execDC(0xc028c06e48?, {0x3c30ef8, 0xc07243d540}, 0xc063893480, 0x440ba8?, {0x33169ff, 0xfe}, {0xc057f0b7a0, 0x9, 0x9})
/usr/local/go/src/database/sql/sql.go:1708 +0x216
database/sql.(*Tx).ExecContext(0xc04a836e00, {0x3c30ef8, 0xc07243d540}, {0x33169ff, 0xfe}, {0xc057f0b7a0, 0x9, 0x9})
/usr/local/go/src/database/sql/sql.go:2516 +0xad
storj.io/storj/shared/tagsql.(*sqlTx).ExecContext(0xc0736a0798, {0x3c30ef8, 0xc07243d540}, {0x33169ff, 0xfe}, {0xc057f0b7a0, 0x9, 0x9})
/go/src/storj.io/storj/shared/tagsql/tx.go:55 +0x306
storj.io/storj/satellite/satellitedb.(*StoragenodeAccounting).SaveRollup.func1.1({0x3c30ef8, 0xc0734c7900}, 0xc028c071e8)
/go/src/storj.io/storj/satellite/satellitedb/storagenodeaccounting.go:320 +0xb2b
storj.io/storj/satellite/satellitedb.(*StoragenodeAccounting).SaveRollup.func1.(*DB).WithTx.4({0x3c30ef8, 0xc073579720}, {0x3c3d3f8, 0xc0736a0798})
/go/src/storj.io/storj/satellite/satellitedb/dbx/gen.go:82 +0x7d
storj.io/storj/shared/dbutil/txutil.withTxOnce({0x3c30ef8, 0xc073579720}, {0x3c4b600, 0xc001d7df40}, 0x0, 0xc028c075f8)
/go/src/storj.io/storj/shared/dbutil/txutil/transactions.go:76 +0x302
storj.io/storj/shared/dbutil/txutil.WithTx({0x3c317b8, 0xc073949a40}, {0x3c4b600, 0xc001d7df40}, 0x0, 0xc028c075f8)
/go/src/storj.io/storj/shared/dbutil/txutil/transactions.go:37 +0x258
storj.io/storj/satellite/satellitedb/dbx.(*DB).WithTx(...)
/go/src/storj.io/storj/satellite/satellitedb/dbx/gen.go:81
storj.io/storj/satellite/satellitedb.(*StoragenodeAccounting).SaveRollup.func1({0x3c30ef8, 0xc0735795e0}, 0xc001d7df40, {0xc03f57b0c0, 0x3e8, 0x155e8})
/go/src/storj.io/storj/satellite/satellitedb/storagenodeaccounting.go:225 +0x279
storj.io/storj/satellite/satellitedb.(*StoragenodeAccounting).SaveRollup(0xc00011bef0, {0x3c30ef8, 0xc035766fa0}, {0xffff62d6dd620000?, 0xa?, 0x0?}, 0xc0021ca870)
/go/src/storj.io/storj/satellite/satellitedb/storagenodeaccounting.go:358 +0x445
storj.io/storj/satellite/accounting/rollup.(*Service).Rollup(0xc0022d0700, {0x3c30ef8, 0xc0014eab40})
/go/src/storj.io/storj/satellite/accounting/rollup/rollup.go:99 +0x60f
storj.io/storj/satellite/accounting/rollup.(*Service).Run.func1({0x3c30ec0?, 0xc00102f2d0?})
/go/src/storj.io/storj/satellite/accounting/rollup/rollup.go:53 +0x45
storj.io/common/sync2.(*Cycle).Run(0xc0022d06c0, {0x3c30ef8, 0xc0014ea960}, 0xc028c07cd0)
/go/pkg/mod/storj.io/common@v0.0.0-20250714141838-91770d7c2a51/sync2/cycle.go:102 +0x19b
storj.io/storj/satellite/accounting/rollup.(*Service).Run(0xc0022d0700, {0x3c30ef8, 0xc0014ea960})
/go/src/storj.io/storj/satellite/accounting/rollup/rollup.go:52 +0x1af
storj.io/storj/private/lifecycle.(*Group).Run.func2.1({0x3c317b8?, 0xc001e0d860?})
/go/src/storj.io/storj/private/lifecycle/group.go:87 +0x2d
runtime/pprof.Do({0x3c30ef8?, 0xc001979e00?}, {{0xc000cf8080?, 0xc002586600?, 0x478319?}}, 0xc0019dbf08)
/usr/local/go/src/runtime/pprof/runtime.go:51 +0x8c
storj.io/storj/private/lifecycle.(*Group).Run.func2()
/go/src/storj.io/storj/private/lifecycle/group.go:86 +0x18e
golang.org/x/sync/errgroup.(*Group).Go.func1()
/go/pkg/mod/golang.org/x/sync@v0.16.0/errgroup/errgroup.go:93 +0x50
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
/go/pkg/mod/golang.org/x/sync@v0.16.0/errgroup/errgroup.go:78 +0x93
The relevant part of the code in grpc-go:
https://github.com/grpc/grpc-go/blob/7f0a3cc04cd2300f6f65803972d46c7dbff027d8/stream.go#L397-L404
It seems to be waiting for the client connection or context to be canceled.
go.mod
module storj.io/storj
go 1.24.0
require (
cloud.google.com/go v0.121.4
cloud.google.com/go/profiler v0.4.0
cloud.google.com/go/secretmanager v1.14.7
cloud.google.com/go/spanner v1.84.0
cloud.google.com/go/storage v1.55.0
github.com/alessio/shellescape v1.2.2
github.com/alicebob/miniredis/v2 v2.13.3
github.com/blang/semver v3.5.1+incompatible
github.com/blang/semver/v4 v4.0.0
github.com/bmkessler/fastdiv v0.0.0-20190227075523-41d5178f2044
github.com/calebcase/tmpfile v1.0.3
github.com/coreos/go-oidc/v3 v3.11.0
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/dgraph-io/badger/v4 v4.5.0
github.com/dsnet/try v0.0.3
github.com/fatih/color v1.15.0
github.com/go-oauth2/oauth2/v4 v4.4.2
github.com/gogo/protobuf v1.3.2
github.com/golang-jwt/jwt v3.2.1+incompatible
github.com/google/go-cmp v0.7.0
github.com/googleapis/go-sql-spanner v1.11.1-0.20250214171559-1bccea5dfec5
github.com/gorilla/mux v1.8.0
github.com/gorilla/schema v1.2.0
github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451
github.com/jackc/pgtype v1.14.1
github.com/jackc/pgx/v5 v5.6.0
github.com/jtolds/monkit-hw/v2 v2.0.0-20250117140252-1a544613ac79
github.com/jtolio/mito v0.0.0-20230523171229-d78ef06bb77b
github.com/jtolio/noiseconn v0.0.0-20230301220541-88105e6c8ac6
github.com/klauspost/compress v1.17.11
github.com/linkedin/goavro/v2 v2.13.1
github.com/loov/hrtime v1.0.3
github.com/mattn/go-sqlite3 v1.14.28
github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce
github.com/nsf/termbox-go v0.0.0-20200418040025-38ba6e5628f1
github.com/oschwald/maxminddb-golang v1.12.0
github.com/pkg/errors v0.9.1
github.com/pquerna/otp v1.3.0
github.com/prometheus/client_golang v1.20.5
github.com/prometheus/common v0.55.0
github.com/redis/go-redis/v9 v9.5.1
github.com/shirou/gopsutil/v3 v3.21.3
github.com/shopspring/decimal v1.2.0
github.com/spacemonkeygo/monkit/v3 v3.0.24
github.com/spacemonkeygo/tlshowdy v0.0.0-20160207005338-8fa2cec1d7cd
github.com/spf13/cobra v1.8.1
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.18.2
github.com/stretchr/testify v1.10.0
github.com/stripe/stripe-go/v81 v81.3.1
github.com/vbauerster/mpb/v8 v8.4.0
github.com/zeebo/assert v1.3.1
github.com/zeebo/blake3 v0.2.3
github.com/zeebo/clingy v0.0.0-20230602044025-906be850f10d
github.com/zeebo/errs v1.4.0
github.com/zeebo/errs/v2 v2.0.5
github.com/zeebo/ini v0.0.0-20210514163846-cc8fbd8d9599
github.com/zeebo/mwc v0.0.6
github.com/zeebo/structs v1.0.3-0.20230601144555-f2db46069602
github.com/zeebo/sudo v1.0.2
github.com/zeebo/xxh3 v1.0.2
github.com/zyedidia/generic v1.2.1
go.etcd.io/bbolt v1.3.5
go.uber.org/mock v0.5.0
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.40.0
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
golang.org/x/net v0.42.0
golang.org/x/oauth2 v0.30.0
golang.org/x/sync v0.16.0
golang.org/x/sys v0.34.0
golang.org/x/term v0.33.0
golang.org/x/text v0.27.0
golang.org/x/time v0.12.0
google.golang.org/api v0.244.0
google.golang.org/grpc v1.74.2
google.golang.org/protobuf v1.36.6
gopkg.in/segmentio/analytics-go.v3 v3.1.0
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.33.3
k8s.io/apimachinery v0.33.3
k8s.io/client-go v0.33.3
storj.io/common v0.0.0-20250730153613-a92febfcb4a1
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156
storj.io/minmaxheap v0.0.0-20250403032542-1e24a6fe9c16
storj.io/monkit-jaeger v0.0.0-20250523220404-454c1b072fad
storj.io/uplink v1.13.2-0.20250731185106-cadfdaef0b55
)
require (
cel.dev/expr v0.24.0 // indirect
cloud.google.com/go/auth v0.16.3 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/bigquery v1.69.0 // indirect
cloud.google.com/go/compute/metadata v0.7.0 // indirect
cloud.google.com/go/iam v1.5.2 // indirect
cloud.google.com/go/longrunning v0.6.7 // indirect
cloud.google.com/go/monitoring v1.24.2 // indirect
github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.3 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
github.com/andybalholm/brotli v1.0.6 // indirect
github.com/apache/arrow/go/v15 v15.0.2 // indirect
github.com/apache/thrift v0.17.0 // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/boombuler/barcode v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect
github.com/dgraph-io/ristretto/v2 v2.0.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/elastic/gosigar v0.14.3 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/flynn/noise v1.0.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v24.3.25+incompatible // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.2 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgx/v4 v4.15.0 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/jtolds/tracetagger/v2 v2.0.0-rc5 // indirect
github.com/jtolio/crawlspace v0.0.0-20231116162947-3ec5cc6b36c5 // indirect
github.com/jtolio/crawlspace/tools v0.0.0-20231116162947-3ec5cc6b36c5 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/quic-go/quic-go v0.53.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.13.1 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tklauser/go-sysconf v0.3.4 // indirect
github.com/tklauser/numcpus v0.2.1 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect
github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb // indirect
github.com/zeebo/admission/v3 v3.0.3 // indirect
github.com/zeebo/float16 v0.1.0 // indirect
github.com/zeebo/goof v0.0.0-20230907150950-e9457bc94477 // indirect
github.com/zeebo/incenc v0.0.0-20180505221441-0d92902eec54 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
go.opentelemetry.io/otel v1.36.0 // indirect
go.opentelemetry.io/otel/metric v1.36.0 // indirect
go.opentelemetry.io/otel/sdk v1.36.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
go.opentelemetry.io/otel/trace v1.36.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/tools v0.34.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250728155136-f173205681a0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
storj.io/infectious v0.0.2 // indirect
storj.io/picobuf v0.0.4 // indirect
)
Expected behavior
The goroutine is cleaned up properly.
Additional context
I'm currently still figuring out a local reproducer.
My best guess the issue has something to do with the #12635 change.
I'm not entirely sure whether this is a latent bug in go-sql-spanner or somewhere in our stack, but the main difference causing the issue was spanner library, a bump from v1.75.0 to v1.84.0.