-
Notifications
You must be signed in to change notification settings - Fork 742
Closed
Labels
affects-7.1This bug affects the 7.1.x(LTS) versions.This bug affects the 7.1.x(LTS) versions.affects-7.5This bug affects the 7.5.x(LTS) versions.This bug affects the 7.5.x(LTS) versions.affects-8.1This bug affects the 8.1.x(LTS) versions.This bug affects the 8.1.x(LTS) versions.affects-8.5This bug affects the 8.5.x(LTS) versions.This bug affects the 8.5.x(LTS) versions.impact/panicseverity/majortype/bugThe issue is confirmed as a bug.The issue is confirmed as a bug.
Description
Bug Report
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x8 pc=0x47288c]
goroutine 31644974457 [running]:
bytes.(*Buffer).Len(...)
/usr/local/go/src/bytes/buffer.go:79
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0x405a28df90, {0x408bece880?, 0x4044348ca8?, 0x40b1744ae0?})
/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/transport.go:166 +0xcc
google.golang.org/grpc/internal/transport.(*transportReader).Read(0x408bece840, {0x408bece880?, 0x40b1744b68?, 0x473d74?})
/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/transport.go:525 +0x34
io.ReadAtLeast({0x1849060, 0x408bece840}, {0x408bece880, 0x5, 0x5}, 0x5)
/usr/local/go/src/io/io.go:335 +0xa0
io.ReadFull(...)
/usr/local/go/src/io/io.go:354
google.golang.org/grpc/internal/transport.(*Stream).Read(0x40989826c0, {0x408bece880, 0x5, 0x5})
/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/transport.go:509 +0x94
google.golang.org/grpc.(*parser).recvMsg(0x408bece870, 0x400000)
/go/pkg/mod/google.golang.org/grpc@v1.62.1/rpc_util.go:614 +0x48
google.golang.org/grpc.recvAndDecompress(0x408bece870, 0x40989826c0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0})
/go/pkg/mod/google.golang.org/grpc@v1.62.1/rpc_util.go:753 +0x58
google.golang.org/grpc.recv(0x1c7daf7?, {0xfffd2c698290, 0x260dfe0}, 0x1be9546?, {0x0?, 0x0?}, {0x14b7f80, 0x40a56238c0}, 0x402209aa80?, 0x14b7e20?, ...)
/go/pkg/mod/google.golang.org/grpc@v1.62.1/rpc_util.go:833 +0x64
google.golang.org/grpc.(*csAttempt).recvMsg(0x40801c21a0, {0x14b7f80?, 0x40a56238c0}, 0x1f1d8?)
/go/pkg/mod/google.golang.org/grpc@v1.62.1/stream.go:1085 +0x208
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x40b1744f38?)
/go/pkg/mod/google.golang.org/grpc@v1.62.1/stream.go:928 +0x28
google.golang.org/grpc.(*clientStream).withRetry(0x4098982480, 0x40b1744fe8, 0x40b1744fd8)
/go/pkg/mod/google.golang.org/grpc@v1.62.1/stream.go:761 +0x4e8
google.golang.org/grpc.(*clientStream).RecvMsg(0x4098982480, {0x14b7f80?, 0x40a56238c0?})
/go/pkg/mod/google.golang.org/grpc@v1.62.1/stream.go:927 +0xc0
github.com/pingcap/kvproto/pkg/tsopb.(*tSOTsoClient).Recv(0x40c92169d0)
/go/pkg/mod/github.com/pingcap/kvproto@v0.0.0-20240403065636-c699538f7aa1/pkg/tsopb/tsopb.pb.go:1011 +0x58
github.com/tikv/pd/server.(*GrpcServer).getGlobalTSO(0x403fe9e4e0, {0x185caa8, 0x40990a6cc0})
/go/src/github.com/tikv/pd/server/forward.go:468 +0x438
github.com/tikv/pd/server.(*GrpcServer).UpdateServiceSafePointV2(0x403fe9e4e0, {0x185caa8, 0x40990a6cc0}, 0x40547bcf60)
/go/src/github.com/tikv/pd/server/gc_service.go:109 +0x9c
github.com/pingcap/kvproto/pkg/pdpb._PD_UpdateServiceSafePointV2_Handler.func1({0x185caa8, 0x40990a6cc0}, {0x14cd340?, 0x40547bcf60})
/go/pkg/mod/github.com/pingcap/kvproto@v0.0.0-20240403065636-c699538f7aa1/pkg/pdpb/pdpb.pb.go:10124 +0x74
github.com/grpc-ecosystem/go-grpc-prometheus.init.(*ServerMetrics).UnaryServerInterceptor.func3({0x185caa8, 0x40990a6cc0}, {0x14cd340, 0x40547bcf60}, 0x0?, 0x40ce2a59e0)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:107 +0x74
go.etcd.io/etcd/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6.1({0x185caa8?, 0x40990a6cc0?}, {0x14cd340?, 0x40547bcf60?})
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.4.0/chain.go:48 +0x50
go.etcd.io/etcd/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func4({0x185caa8, 0x40990a6cc0}, {0x14cd340, 0x40547bcf60}, 0x40b1745798?, 0x40a5623240)
/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca/etcdserver/api/v3rpc/interceptor.go:71 +0x19c
go.etcd.io/etcd/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6.1({0x185caa8?, 0x40990a6cc0?}, {0x14cd340?, 0x40547bcf60?})
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.4.0/chain.go:48 +0x50
go.etcd.io/etcd/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func3({0x185caa8?, 0x40990a6cc0}, {0x14cd340?, 0x40547bcf60}, 0x4062e1e980, 0x40a5623280)
/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca/etcdserver/api/v3rpc/interceptor.go:78 +0x88
go.etcd.io/etcd/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6({0x185caa8, 0x40990a6cc0}, {0x14cd340, 0x40547bcf60}, 0x4062e1e980, 0x4da90c?)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.4.0/chain.go:53 +0x128
github.com/pingcap/kvproto/pkg/pdpb._PD_UpdateServiceSafePointV2_Handler({0x1552140?, 0x403fe9e4e0}, {0x185caa8, 0x40990a6cc0}, 0x4073cfe080, 0x403d55bec0)
/go/pkg/mod/github.com/pingcap/kvproto@v0.0.0-20240403065636-c699538f7aa1/pkg/pdpb/pdpb.pb.go:10126 +0x12c
google.golang.org/grpc.(*Server).processUnaryRPC(0x40436ddc00, {0x185caa8, 0x40ea13be90}, {0x1867640, 0x4089106b60}, 0x401f045b00, 0x403feb2660, 0x25c4d90, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1386 +0xba0
google.golang.org/grpc.(*Server).handleStream(0x40436ddc00, {0x1867640, 0x4089106b60}, 0x401f045b00)
/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1797 +0xc70
google.golang.org/grpc.(*Server).serveStreams.func2.1()
/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1027 +0x8c
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 31644974454
/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1038 +0x150
After confirming with @ystaticy, it's possible to call UpdateServiceSafePointV2
concurrently, which results in calling Recv
concurrently on a stream.
Metadata
Metadata
Assignees
Labels
affects-7.1This bug affects the 7.1.x(LTS) versions.This bug affects the 7.1.x(LTS) versions.affects-7.5This bug affects the 7.5.x(LTS) versions.This bug affects the 7.5.x(LTS) versions.affects-8.1This bug affects the 8.1.x(LTS) versions.This bug affects the 8.1.x(LTS) versions.affects-8.5This bug affects the 8.5.x(LTS) versions.This bug affects the 8.5.x(LTS) versions.impact/panicseverity/majortype/bugThe issue is confirmed as a bug.The issue is confirmed as a bug.