Skip to content

XdsTest/DropTest.Update/FakeResolver WARNING: ThreadSanitizer: use of an invalid mutex #20928

@apolcyn

Description

@apolcyn

Seen in run of test/cpp/end2end/xds_end2end_test@poller=epollex.test/cpp/end2end/xds_end2end_test@poller=epollex

in https://source.cloud.google.com/results/invocations/d69e7cd6-98bc-4589-b21d-ccbcf180a83a/targets/%2F%2Ftest%2Fcpp%2Fend2end:xds_end2end_test@poller%3Depollex/tests
in PR: #20596

from the log:

I1105 00:55:23.992859632      36 subchannel.cc:940]          Subchannel 0x7b5400005280: Retry in 991 milliseconds
I1105 00:55:23.994515647      17 xds_end2end_test.cc:942]    Balancer shutdown completed
[       OK ] XdsTest/DropTest.Update/FakeResolver (96224 ms)
[ RUN      ] XdsTest/DropTest.Update/XdsResolverWithLoadReporting
I1105 00:55:23.999869596      17 xds_end2end_test.cc:905]    starting Backend server on port 23025
==================
WARNING: ThreadSanitizer: use of an invalid mutex (e.g. uninitialized or destroyed) (pid=17)
    #0 pthread_mutex_lock /tmp/clang-build/src/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_common_interceptors.inc:4141:3 (xds_end2end_test+0x4b3966)
    #1 gpr_mu_lock /proc/self/cwd/src/core/lib/gpr/sync_posix.cc:65:3 (liblibgpr_Ubase.so+0xe552)
    #2 pss_lock_adam(grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/ev_epollex_linux.cc:1304:3 (liblibgrpc_Ubase_Uc.so+0xb630f)
    #3 pollset_set_add_fd(grpc_pollset_set*, grpc_fd*) /proc/self/cwd/src/core/lib/iomgr/ev_epollex_linux.cc:1348:9 (liblibgrpc_Ubase_Uc.so+0xb0d14)
    #4 grpc_pollset_set_add_fd(grpc_pollset_set*, grpc_fd*) /proc/self/cwd/src/core/lib/iomgr/ev_posix.cc:396:3 (liblibgrpc_Ubase_Uc.so+0xc28dc)
    #5 tcp_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/tcp_posix.cc:1116:3 (liblibgrpc_Ubase_Uc.so+0xe5fb3)
    #6 grpc_endpoint_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/endpoint.cc:41:3 (liblibgrpc_Ubase_Uc.so+0x9e657)
    #7 endpoint_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/security/transport/secure_endpoint.cc:394:3 (liblibgrpc_Usecure.so+0x81e8b)
    #8 grpc_endpoint_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/endpoint.cc:41:3 (liblibgrpc_Ubase_Uc.so+0x9e657)
    #9 set_pollset_set(grpc_transport*, grpc_stream*, grpc_pollset_set*) /proc/self/cwd/src/core/ext/transport/chttp2/transport/chttp2_transport.cc:2921:3 (liblibgrpc_Utransport_Uchttp2.so+0x35934)
    #10 grpc_transport_set_pops(grpc_transport*, grpc_stream*, grpc_polling_entity*) /proc/self/cwd/src/core/lib/transport/transport.cc:152:5 (liblibgrpc_Ubase_Uc.so+0x163bcc)
    #11 set_pollset_or_pollset_set(grpc_call_element*, grpc_polling_entity*) /proc/self/cwd/src/core/lib/channel/connected_channel.cc:164:3 (liblibgrpc_Ubase_Uc.so+0x7ffef)
    #12 grpc_call_stack_set_pollset_or_pollset_set(grpc_call_stack*, grpc_polling_entity*) /proc/self/cwd/src/core/lib/channel/channel_stack.cc:201:5 (liblibgrpc_Ubase_Uc.so+0x63c33)
    #13 grpc_core::SubchannelCall::SubchannelCall(grpc_core::SubchannelCall::Args, grpc_error**) /proc/self/cwd/src/core/ext/filters/client_channel/subchannel.cc:168:3 (liblibgrpc_Uclient_Uchannel.so+0xeefbf)
    #14 grpc_core::SubchannelCall::Create(grpc_core::SubchannelCall::Args, grpc_error**) /proc/self/cwd/src/core/ext/filters/client_channel/subchannel.cc:144:38 (liblibgrpc_Uclient_Uchannel.so+0xeebb9)
    #15 grpc_core::(anonymous namespace)::CallData::CreateSubchannelCall(grpc_call_element*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:3664:22 (liblibgrpc_Uclient_Uchannel.so+0x8d4b5)
    #16 grpc_core::(anonymous namespace)::CallData::PickDone(void*, grpc_error*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:3698:10 (liblibgrpc_Uclient_Uchannel.so+0x8bb70)
    #17 grpc_core::(anonymous namespace)::CallData::PickSubchannel(void*, grpc_error*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:3853:5 (liblibgrpc_Uclient_Uchannel.so+0x80c94)
    #18 grpc_core::(anonymous namespace)::CallData::StartTransportStreamOpBatch(grpc_call_element*, grpc_transport_stream_op_batch*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:2138:5 (liblibgrpc_Uclient_Uchannel.so+0x7e4fb)
    #19 grpc_call_next_op(grpc_call_element*, grpc_transport_stream_op_batch*) /proc/self/cwd/src/core/lib/channel/channel_stack.cc:227:3 (liblibgrpc_Ubase_Uc.so+0x63ec4)
    #20 execute_batch_in_call_combiner(void*, grpc_error*) /proc/self/cwd/src/core/lib/surface/call.cc:627:3 (liblibgrpc_Ubase_Uc.so+0x12347d)
    #21 exec_ctx_run(grpc_closure*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:40:3 (liblibgrpc_Ubase_Uc.so+0xc3d87)
    #22 grpc_closure_run(char const*, int, grpc_closure*, grpc_error*) /proc/self/cwd/./src/core/lib/iomgr/closure.h:264:5 (liblibgrpc_Umessage_Usize_Ufilter.so+0xbdee)
    #23 grpc_core::CallCombiner::TsanClosure(void*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/call_combiner.cc:83:3 (liblibgrpc_Ubase_Uc.so+0x998c8)
    #24 exec_ctx_run(grpc_closure*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:40:3 (liblibgrpc_Ubase_Uc.so+0xc3d87)
    #25 grpc_core::ExecCtx::Flush() /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:158:9 (liblibgrpc_Ubase_Uc.so+0xc3ace)
    #26 pollset_work(grpc_pollset*, grpc_pollset_worker**, long) /proc/self/cwd/src/core/lib/iomgr/ev_epollex_linux.cc:1134:34 (liblibgrpc_Ubase_Uc.so+0xae2bb)
    #27 pollset_work(grpc_pollset*, grpc_pollset_worker**, long) /proc/self/cwd/src/core/lib/iomgr/ev_posix.cc:324:21 (liblibgrpc_Ubase_Uc.so+0xc2120)
    #28 grpc_pollset_work(grpc_pollset*, grpc_pollset_worker**, long) /proc/self/cwd/src/core/lib/iomgr/pollset.cc:48:10 (liblibgrpc_Ubase_Uc.so+0xcfd6a)
    #29 run_poller(void*, grpc_error*) /proc/self/cwd/src/core/ext/filters/client_channel/backup_poller.cc:130:7 (liblibgrpc_Uclient_Uchannel.so+0x7ac03)
    #30 exec_ctx_run(grpc_closure*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:40:3 (liblibgrpc_Ubase_Uc.so+0xc3d87)
    #31 grpc_core::ExecCtx::Flush() /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:158:9 (liblibgrpc_Ubase_Uc.so+0xc3ace)
    #32 run_some_timers() /proc/self/cwd/src/core/lib/iomgr/timer_manager.cc:134:30 (liblibgrpc_Ubase_Uc.so+0xfca67)
    #33 timer_main_loop() /proc/self/cwd/src/core/lib/iomgr/timer_manager.cc:237:9 (liblibgrpc_Ubase_Uc.so+0xfc683)
    #34 timer_thread(void*) /proc/self/cwd/src/core/lib/iomgr/timer_manager.cc:284:3 (liblibgrpc_Ubase_Uc.so+0xfc57a)
    #35 grpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::operator()(void*) const /proc/self/cwd/src/core/lib/gprpp/thd_posix.cc:139:27 (liblibgpr_Ubase.so+0x16f48)
    #36 grpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::__invoke(void*) /proc/self/cwd/src/core/lib/gprpp/thd_posix.cc:109:25 (liblibgpr_Ubase.so+0x16d68)

  Mutex M37228230 (0x7b1c00021b80) created at:
    #0 pthread_mutex_lock /tmp/clang-build/src/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_common_interceptors.inc:4141:3 (xds_end2end_test+0x4b3966)
    #1 gpr_mu_lock /proc/self/cwd/src/core/lib/gpr/sync_posix.cc:65:3 (liblibgpr_Ubase.so+0xe552)
    #2 pss_lock_adam(grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/ev_epollex_linux.cc:1304:3 (liblibgrpc_Ubase_Uc.so+0xb630f)
    #3 pollset_set_add_fd(grpc_pollset_set*, grpc_fd*) /proc/self/cwd/src/core/lib/iomgr/ev_epollex_linux.cc:1348:9 (liblibgrpc_Ubase_Uc.so+0xb0d14)
    #4 grpc_pollset_set_add_fd(grpc_pollset_set*, grpc_fd*) /proc/self/cwd/src/core/lib/iomgr/ev_posix.cc:396:3 (liblibgrpc_Ubase_Uc.so+0xc28dc)
    #5 tcp_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/tcp_posix.cc:1116:3 (liblibgrpc_Ubase_Uc.so+0xe5fb3)
    #6 grpc_endpoint_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/endpoint.cc:41:3 (liblibgrpc_Ubase_Uc.so+0x9e657)
    #7 endpoint_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/security/transport/secure_endpoint.cc:394:3 (liblibgrpc_Usecure.so+0x81e8b)
    #8 grpc_endpoint_add_to_pollset_set(grpc_endpoint*, grpc_pollset_set*) /proc/self/cwd/src/core/lib/iomgr/endpoint.cc:41:3 (liblibgrpc_Ubase_Uc.so+0x9e657)
    #9 set_pollset_set(grpc_transport*, grpc_stream*, grpc_pollset_set*) /proc/self/cwd/src/core/ext/transport/chttp2/transport/chttp2_transport.cc:2921:3 (liblibgrpc_Utransport_Uchttp2.so+0x35934)
    #10 grpc_transport_set_pops(grpc_transport*, grpc_stream*, grpc_polling_entity*) /proc/self/cwd/src/core/lib/transport/transport.cc:152:5 (liblibgrpc_Ubase_Uc.so+0x163bcc)
    #11 set_pollset_or_pollset_set(grpc_call_element*, grpc_polling_entity*) /proc/self/cwd/src/core/lib/channel/connected_channel.cc:164:3 (liblibgrpc_Ubase_Uc.so+0x7ffef)
    #12 grpc_call_stack_set_pollset_or_pollset_set(grpc_call_stack*, grpc_polling_entity*) /proc/self/cwd/src/core/lib/channel/channel_stack.cc:201:5 (liblibgrpc_Ubase_Uc.so+0x63c33)
    #13 grpc_core::SubchannelCall::SubchannelCall(grpc_core::SubchannelCall::Args, grpc_error**) /proc/self/cwd/src/core/ext/filters/client_channel/subchannel.cc:168:3 (liblibgrpc_Uclient_Uchannel.so+0xeefbf)
    #14 grpc_core::SubchannelCall::Create(grpc_core::SubchannelCall::Args, grpc_error**) /proc/self/cwd/src/core/ext/filters/client_channel/subchannel.cc:144:38 (liblibgrpc_Uclient_Uchannel.so+0xeebb9)
    #15 grpc_core::(anonymous namespace)::CallData::CreateSubchannelCall(grpc_call_element*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:3664:22 (liblibgrpc_Uclient_Uchannel.so+0x8d4b5)
    #16 grpc_core::(anonymous namespace)::CallData::PickDone(void*, grpc_error*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:3698:10 (liblibgrpc_Uclient_Uchannel.so+0x8bb70)
    #17 grpc_core::(anonymous namespace)::CallData::PickSubchannel(void*, grpc_error*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:3853:5 (liblibgrpc_Uclient_Uchannel.so+0x80c94)
    #18 grpc_core::(anonymous namespace)::CallData::StartTransportStreamOpBatch(grpc_call_element*, grpc_transport_stream_op_batch*) /proc/self/cwd/src/core/ext/filters/client_channel/client_channel.cc:2138:5 (liblibgrpc_Uclient_Uchannel.so+0x7e4fb)
    #19 grpc_call_next_op(grpc_call_element*, grpc_transport_stream_op_batch*) /proc/self/cwd/src/core/lib/channel/channel_stack.cc:227:3 (liblibgrpc_Ubase_Uc.so+0x63ec4)
    #20 execute_batch_in_call_combiner(void*, grpc_error*) /proc/self/cwd/src/core/lib/surface/call.cc:627:3 (liblibgrpc_Ubase_Uc.so+0x12347d)
    #21 exec_ctx_run(grpc_closure*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:40:3 (liblibgrpc_Ubase_Uc.so+0xc3d87)
    #22 grpc_closure_run(char const*, int, grpc_closure*, grpc_error*) /proc/self/cwd/./src/core/lib/iomgr/closure.h:264:5 (liblibgrpc_Umessage_Usize_Ufilter.so+0xbdee)
    #23 grpc_core::CallCombiner::TsanClosure(void*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/call_combiner.cc:83:3 (liblibgrpc_Ubase_Uc.so+0x998c8)
    #24 exec_ctx_run(grpc_closure*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:40:3 (liblibgrpc_Ubase_Uc.so+0xc3d87)
    #25 grpc_core::ExecCtx::Flush() /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:158:9 (liblibgrpc_Ubase_Uc.so+0xc3ace)
    #26 pollset_work(grpc_pollset*, grpc_pollset_worker**, long) /proc/self/cwd/src/core/lib/iomgr/ev_epollex_linux.cc:1134:34 (liblibgrpc_Ubase_Uc.so+0xae2bb)
    #27 pollset_work(grpc_pollset*, grpc_pollset_worker**, long) /proc/self/cwd/src/core/lib/iomgr/ev_posix.cc:324:21 (liblibgrpc_Ubase_Uc.so+0xc2120)
    #28 grpc_pollset_work(grpc_pollset*, grpc_pollset_worker**, long) /proc/self/cwd/src/core/lib/iomgr/pollset.cc:48:10 (liblibgrpc_Ubase_Uc.so+0xcfd6a)
    #29 run_poller(void*, grpc_error*) /proc/self/cwd/src/core/ext/filters/client_channel/backup_poller.cc:130:7 (liblibgrpc_Uclient_Uchannel.so+0x7ac03)
    #30 exec_ctx_run(grpc_closure*, grpc_error*) /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:40:3 (liblibgrpc_Ubase_Uc.so+0xc3d87)
    #31 grpc_core::ExecCtx::Flush() /proc/self/cwd/src/core/lib/iomgr/exec_ctx.cc:158:9 (liblibgrpc_Ubase_Uc.so+0xc3ace)
    #32 run_some_timers() /proc/self/cwd/src/core/lib/iomgr/timer_manager.cc:134:30 (liblibgrpc_Ubase_Uc.so+0xfca67)
    #33 timer_main_loop() /proc/self/cwd/src/core/lib/iomgr/timer_manager.cc:237:9 (liblibgrpc_Ubase_Uc.so+0xfc683)
    #34 timer_thread(void*) /proc/self/cwd/src/core/lib/iomgr/timer_manager.cc:284:3 (liblibgrpc_Ubase_Uc.so+0xfc57a)
    #35 grpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::operator()(void*) const /proc/self/cwd/src/core/lib/gprpp/thd_posix.cc:139:27 (liblibgpr_Ubase.so+0x16f48)
    #36 grpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&)::'lambda'(void*)::__invoke(void*) /proc/self/cwd/src/core/lib/gprpp/thd_posix.cc:109:25 (liblibgpr_Ubase.so+0x16d68)

SUMMARY: ThreadSanitizer: use of an invalid mutex (e.g. uninitialized or destroyed) /proc/self/cwd/src/core/lib/gpr/sync_posix.cc:65:3 in gpr_mu_lock
==================

Metadata

Metadata

Assignees

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions