-
Notifications
You must be signed in to change notification settings - Fork 10.9k
Closed
Description
Seen in run of test/cpp/end2end/xds_end2end_test@poller=epollex.test/cpp/end2end/xds_end2end_test@poller=epollex
in https://source.cloud.google.com/results/invocations/d69e7cd6-98bc-4589-b21d-ccbcf180a83a/targets/%2F%2Ftest%2Fcpp%2Fend2end:xds_end2end_test@poller%3Depollex/tests
in PR: #20596
from the log:
I1105 00:55:23.992859632 36 subchannel.cc:940] Subchannel 0x7b5400005280: Retry in 991 milliseconds
I1105 00:55:23.994515647 17 xds_end2end_test.cc:942] Balancer shutdown completed
[ OK ] XdsTest/DropTest.Update/FakeResolver (96224 ms)
[ RUN ] XdsTest/DropTest.Update/XdsResolverWithLoadReporting
I1105 00:55:23.999869596 17 xds_end2end_test.cc:905] starting Backend server on port 23025
==================
WARNING: ThreadSanitizer: use of an invalid mutex (e.g. uninitialized or destroyed) (pid=17)
#0 pthread_mutex_lock /tmp/clang-build/src/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_common_interceptors.inc:4141:3 (xds_end2end_test+0x4b3966)
#1 gpr_mu_lock /proc/self/cwd/src/core/lib/gpr/sync_posix.cc:65:3 (liblibgpr_Ubase.so+0xe552)
#2 pss_lock_adam(grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/ev_epollex_linux.cc:1304:3 (liblibgrpc_Ubase_Uc.so+0xb630f)
#3 pollset_set_add_fd(grpc_pollset_set*, grpc_fd*) /proc/self/cwd/src/core/lib/iomgr/ev_epollex_linux.cc:1348:9 (liblibgrpc_Ubase_Uc.so+0xb0d14)
#4 grpc_pollset_set_add_fd(grpc_pollset_set*, grpc_fd*) /proc/self/cwd/src/core/lib/iomgr/ev_posix.cc:396:3 (liblibgrpc_Ubase_Uc.so+0xc28dc)
#5 tcp_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/tcp_posix.cc:1116:3 (liblibgrpc_Ubase_Uc.so+0xe5fb3)
#6 grpc_endpoint_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/endpoint.cc:41:3 (liblibgrpc_Ubase_Uc.so+0x9e657)
#7 endpoint_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/security/transport/secure_endpoint.cc:394:3 (liblibgrpc_Usecure.so+0x81e8b)
#8 grpc_endpoint_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/endpoint.cc:41:3 (liblibgrpc_Ubase_Uc.so+0x9e657)
#9 set_pollset_set(grpc_transport*, grpc_stream*, grpc_pollset_set*) /proc/self/cwd/src/core/ext/transport/chttp2/transport/chttp2_transport.cc:2921:3 (liblibgrpc_Utransport_Uchttp2.so+0x35934)
#10 grpc_transport_set_pops(grpc_transport*, grpc_stream*, grpc_polling_entity*) /proc/self/cwd/src/core/lib/transport/transport.cc:152:5 (liblibgrpc_Ubase_Uc.so+0x163bcc)
#11 set_pollset_or_pollset_set(grpc_call_element*, grpc_polling_entity*) /proc/self/cwd/src/core/lib/channel/connected_channel.cc:164:3 (liblibgrpc_Ubase_Uc.so+0x7ffef)
#12 grpc_call_stack_set_pollset_or_pollset_set(grpc_call_stack*, grpc_polling_entity*) /proc/self/cwd/src/core/lib/channel/channel_stack.cc:201:5 (liblibgrpc_Ubase_Uc.so+0x63c33)
#13 grpc_core::SubchannelCall::SubchannelCall(grpc_core::SubchannelCall::Args, grpc_error**) /proc/self/cwd/src/core/ext/filters/client_channel/subchannel.cc:168:3 (liblibgrpc_Uclient_Uchannel.so+0xeefbf)
#14 grpc_core::SubchannelCall::Create(grpc_core::SubchannelCall::Args, grpc_error**) /proc/self/cwd/src/core/ext/filters/client_channel/subchannel.cc:144:38 (liblibgrpc_Uclient_Uchannel.so+0xeebb9)
#15 grpc_core::(anonymous namespace)::CallData::CreateSubchannelCall(grpc_call_element*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:3664:22 (liblibgrpc_Uclient_Uchannel.so+0x8d4b5)
#16 grpc_core::(anonymous namespace)::CallData::PickDone(void*, grpc_error*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:3698:10 (liblibgrpc_Uclient_Uchannel.so+0x8bb70)
#17 grpc_core::(anonymous namespace)::CallData::PickSubchannel(void*, grpc_error*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:3853:5 (liblibgrpc_Uclient_Uchannel.so+0x80c94)
#18 grpc_core::(anonymous namespace)::CallData::StartTransportStreamOpBatch(grpc_call_element*, grpc_transport_stream_op_batch*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:2138:5 (liblibgrpc_Uclient_Uchannel.so+0x7e4fb)
#19 grpc_call_next_op(grpc_call_element*, grpc_transport_stream_op_batch*) /proc/self/cwd/src/core/lib/channel/channel_stack.cc:227:3 (liblibgrpc_Ubase_Uc.so+0x63ec4)
#20 execute_batch_in_call_combiner(void*, grpc_error*) /proc/self/cwd/src/core/lib/surface/call.cc:627:3 (liblibgrpc_Ubase_Uc.so+0x12347d)
#21 exec_ctx_run(grpc_closure*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:40:3 (liblibgrpc_Ubase_Uc.so+0xc3d87)
#22 grpc_closure_run(char const*, int, grpc_closure*, grpc_error*) /proc/self/cwd/./src/core/lib/iomgr/closure.h:264:5 (liblibgrpc_Umessage_Usize_Ufilter.so+0xbdee)
#23 grpc_core::CallCombiner::TsanClosure(void*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/call_combiner.cc:83:3 (liblibgrpc_Ubase_Uc.so+0x998c8)
#24 exec_ctx_run(grpc_closure*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:40:3 (liblibgrpc_Ubase_Uc.so+0xc3d87)
#25 grpc_core::ExecCtx::Flush() /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:158:9 (liblibgrpc_Ubase_Uc.so+0xc3ace)
#26 pollset_work(grpc_pollset*, grpc_pollset_worker**, long) /proc/self/cwd/src/core/lib/iomgr/ev_epollex_linux.cc:1134:34 (liblibgrpc_Ubase_Uc.so+0xae2bb)
#27 pollset_work(grpc_pollset*, grpc_pollset_worker**, long) /proc/self/cwd/src/core/lib/iomgr/ev_posix.cc:324:21 (liblibgrpc_Ubase_Uc.so+0xc2120)
#28 grpc_pollset_work(grpc_pollset*, grpc_pollset_worker**, long) /proc/self/cwd/src/core/lib/iomgr/pollset.cc:48:10 (liblibgrpc_Ubase_Uc.so+0xcfd6a)
#29 run_poller(void*, grpc_error*) /proc/self/cwd/src/core/ext/filters/client_channel/backup_poller.cc:130:7 (liblibgrpc_Uclient_Uchannel.so+0x7ac03)
#30 exec_ctx_run(grpc_closure*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:40:3 (liblibgrpc_Ubase_Uc.so+0xc3d87)
#31 grpc_core::ExecCtx::Flush() /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:158:9 (liblibgrpc_Ubase_Uc.so+0xc3ace)
#32 run_some_timers() /proc/self/cwd/src/core/lib/iomgr/timer_manager.cc:134:30 (liblibgrpc_Ubase_Uc.so+0xfca67)
#33 timer_main_loop() /proc/self/cwd/src/core/lib/iomgr/timer_manager.cc:237:9 (liblibgrpc_Ubase_Uc.so+0xfc683)
#34 timer_thread(void*) /proc/self/cwd/src/core/lib/iomgr/timer_manager.cc:284:3 (liblibgrpc_Ubase_Uc.so+0xfc57a)
#35 grpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::operator()(void*) const /proc/self/cwd/src/core/lib/gprpp/thd_posix.cc:139:27 (liblibgpr_Ubase.so+0x16f48)
#36 grpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::__invoke(void*) /proc/self/cwd/src/core/lib/gprpp/thd_posix.cc:109:25 (liblibgpr_Ubase.so+0x16d68)
Mutex M37228230 (0x7b1c00021b80) created at:
#0 pthread_mutex_lock /tmp/clang-build/src/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_common_interceptors.inc:4141:3 (xds_end2end_test+0x4b3966)
#1 gpr_mu_lock /proc/self/cwd/src/core/lib/gpr/sync_posix.cc:65:3 (liblibgpr_Ubase.so+0xe552)
#2 pss_lock_adam(grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/ev_epollex_linux.cc:1304:3 (liblibgrpc_Ubase_Uc.so+0xb630f)
#3 pollset_set_add_fd(grpc_pollset_set*, grpc_fd*) /proc/self/cwd/src/core/lib/iomgr/ev_epollex_linux.cc:1348:9 (liblibgrpc_Ubase_Uc.so+0xb0d14)
#4 grpc_pollset_set_add_fd(grpc_pollset_set*, grpc_fd*) /proc/self/cwd/src/core/lib/iomgr/ev_posix.cc:396:3 (liblibgrpc_Ubase_Uc.so+0xc28dc)
#5 tcp_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/tcp_posix.cc:1116:3 (liblibgrpc_Ubase_Uc.so+0xe5fb3)
#6 grpc_endpoint_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/endpoint.cc:41:3 (liblibgrpc_Ubase_Uc.so+0x9e657)
#7 endpoint_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/security/transport/secure_endpoint.cc:394:3 (liblibgrpc_Usecure.so+0x81e8b)
#8 grpc_endpoint_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/endpoint.cc:41:3 (liblibgrpc_Ubase_Uc.so+0x9e657)
#9 set_pollset_set(grpc_transport*, grpc_stream*, grpc_pollset_set*) /proc/self/cwd/src/core/ext/transport/chttp2/transport/chttp2_transport.cc:2921:3 (liblibgrpc_Utransport_Uchttp2.so+0x35934)
#10 grpc_transport_set_pops(grpc_transport*, grpc_stream*, grpc_polling_entity*) /proc/self/cwd/src/core/lib/transport/transport.cc:152:5 (liblibgrpc_Ubase_Uc.so+0x163bcc)
#11 set_pollset_or_pollset_set(grpc_call_element*, grpc_polling_entity*) /proc/self/cwd/src/core/lib/channel/connected_channel.cc:164:3 (liblibgrpc_Ubase_Uc.so+0x7ffef)
#12 grpc_call_stack_set_pollset_or_pollset_set(grpc_call_stack*, grpc_polling_entity*) /proc/self/cwd/src/core/lib/channel/channel_stack.cc:201:5 (liblibgrpc_Ubase_Uc.so+0x63c33)
#13 grpc_core::SubchannelCall::SubchannelCall(grpc_core::SubchannelCall::Args, grpc_error**) /proc/self/cwd/src/core/ext/filters/client_channel/subchannel.cc:168:3 (liblibgrpc_Uclient_Uchannel.so+0xeefbf)
#14 grpc_core::SubchannelCall::Create(grpc_core::SubchannelCall::Args, grpc_error**) /proc/self/cwd/src/core/ext/filters/client_channel/subchannel.cc:144:38 (liblibgrpc_Uclient_Uchannel.so+0xeebb9)
#15 grpc_core::(anonymous namespace)::CallData::CreateSubchannelCall(grpc_call_element*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:3664:22 (liblibgrpc_Uclient_Uchannel.so+0x8d4b5)
#16 grpc_core::(anonymous namespace)::CallData::PickDone(void*, grpc_error*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:3698:10 (liblibgrpc_Uclient_Uchannel.so+0x8bb70)
#17 grpc_core::(anonymous namespace)::CallData::PickSubchannel(void*, grpc_error*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:3853:5 (liblibgrpc_Uclient_Uchannel.so+0x80c94)
#18 grpc_core::(anonymous namespace)::CallData::StartTransportStreamOpBatch(grpc_call_element*, grpc_transport_stream_op_batch*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:2138:5 (liblibgrpc_Uclient_Uchannel.so+0x7e4fb)
#19 grpc_call_next_op(grpc_call_element*, grpc_transport_stream_op_batch*) /proc/self/cwd/src/core/lib/channel/channel_stack.cc:227:3 (liblibgrpc_Ubase_Uc.so+0x63ec4)
#20 execute_batch_in_call_combiner(void*, grpc_error*) /proc/self/cwd/src/core/lib/surface/call.cc:627:3 (liblibgrpc_Ubase_Uc.so+0x12347d)
#21 exec_ctx_run(grpc_closure*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:40:3 (liblibgrpc_Ubase_Uc.so+0xc3d87)
#22 grpc_closure_run(char const*, int, grpc_closure*, grpc_error*) /proc/self/cwd/./src/core/lib/iomgr/closure.h:264:5 (liblibgrpc_Umessage_Usize_Ufilter.so+0xbdee)
#23 grpc_core::CallCombiner::TsanClosure(void*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/call_combiner.cc:83:3 (liblibgrpc_Ubase_Uc.so+0x998c8)
#24 exec_ctx_run(grpc_closure*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:40:3 (liblibgrpc_Ubase_Uc.so+0xc3d87)
#25 grpc_core::ExecCtx::Flush() /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:158:9 (liblibgrpc_Ubase_Uc.so+0xc3ace)
#26 pollset_work(grpc_pollset*, grpc_pollset_worker**, long) /proc/self/cwd/src/core/lib/iomgr/ev_epollex_linux.cc:1134:34 (liblibgrpc_Ubase_Uc.so+0xae2bb)
#27 pollset_work(grpc_pollset*, grpc_pollset_worker**, long) /proc/self/cwd/src/core/lib/iomgr/ev_posix.cc:324:21 (liblibgrpc_Ubase_Uc.so+0xc2120)
#28 grpc_pollset_work(grpc_pollset*, grpc_pollset_worker**, long) /proc/self/cwd/src/core/lib/iomgr/pollset.cc:48:10 (liblibgrpc_Ubase_Uc.so+0xcfd6a)
#29 run_poller(void*, grpc_error*) /proc/self/cwd/src/core/ext/filters/client_channel/backup_poller.cc:130:7 (liblibgrpc_Uclient_Uchannel.so+0x7ac03)
#30 exec_ctx_run(grpc_closure*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:40:3 (liblibgrpc_Ubase_Uc.so+0xc3d87)
#31 grpc_core::ExecCtx::Flush() /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:158:9 (liblibgrpc_Ubase_Uc.so+0xc3ace)
#32 run_some_timers() /proc/self/cwd/src/core/lib/iomgr/timer_manager.cc:134:30 (liblibgrpc_Ubase_Uc.so+0xfca67)
#33 timer_main_loop() /proc/self/cwd/src/core/lib/iomgr/timer_manager.cc:237:9 (liblibgrpc_Ubase_Uc.so+0xfc683)
#34 timer_thread(void*) /proc/self/cwd/src/core/lib/iomgr/timer_manager.cc:284:3 (liblibgrpc_Ubase_Uc.so+0xfc57a)
#35 grpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::operator()(void*) const /proc/self/cwd/src/core/lib/gprpp/thd_posix.cc:139:27 (liblibgpr_Ubase.so+0x16f48)
#36 grpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::__invoke(void*) /proc/self/cwd/src/core/lib/gprpp/thd_posix.cc:109:25 (liblibgpr_Ubase.so+0x16d68)
SUMMARY: ThreadSanitizer: use of an invalid mutex (e.g. uninitialized or destroyed) /proc/self/cwd/src/core/lib/gpr/sync_posix.cc:65:3 in gpr_mu_lock
==================