<table border="1" cellspacing="0" cellpadding="8">
<tr>
<th>Issue</th>
<td>
<a href=https://github.com/llvm/llvm-project/issues/58236>58236</a>
</td>
</tr>
<tr>
<th>Summary</th>
<td>
DebuggerStats::ReportStatistics can cause lock inversion deadlocks if run while targets are processing dyld load notification
</td>
</tr>
<tr>
<th>Labels</th>
<td>
new issue
</td>
</tr>
<tr>
<th>Assignees</th>
<td>
clayborg
</td>
</tr>
<tr>
<th>Reporter</th>
<td>
jimingham
</td>
</tr>
</table>
<pre>
DebuggerStats::ReportStatistics takes the lock returned by:
Module::GetAllocationModuleCollectionMutex
and holds it while it iterates through all the extant modules - thus acquiring each of them in turn. But if another module is right in the middle of creating itself, it looks like it can end up having already taken its Module mutex, and then trying to get the AllocationModuleCollectionMutex. That shouldn't happen for simple Modules, but the two-level creation of modules that happens when using DWARFDebugMap symbol plugin allows the containing module to be visible to ReportStatistics even though it's still in the process of making new sub-modules for the .o files. That stalls against ReportStatistics, and then ReportStatistics stalls getting the Module mutex for one of these in process of being created modules and everything grinds to a halt.
Here's some sample stacks from a session in which this deadlock was observed. These will look a little odd because of the references to lldb-rpc-server, that's just a wrapper around the LLDB framework that Xcode uses.
Thread 0x2c4a28 Thread name "<lldb.process.internal-state(pid=12720)>" 770 samples (1-770) priority 31 (base 31)
770 thread_start + 8 (libsystem_pthread.dylib + 7724) [0x19aeece2c]
770 _pthread_start + 148 (pthread.c:893 in libsystem_pthread.dylib + 28780) [0x19aef206c]
770 lldb_private::HostThreadMacOSX::ThreadCreateTrampoline(void*) + 32 (LLDB + 62542236) [0x1167c119c]
770 lldb_private::HostNativeThreadBase::ThreadCreateTrampoline(void*) + 100 (LLDB + 2620756) [0x112e9bd54]
770 std::1::function::__func<lldb_private::Process::StartPrivateStateThread(bool)::$_6, std::__1::allocator<lldb_private::Process::StartPrivateStateThread(bool)::$_6>, void* ()>::operator()() + 24 (LLDB + 3373800) [0x112f53ae8]
770 lldb_private::Process::RunPrivateStateThread(bool) + 1180 (LLDB + 3346788) [0x112f4d164]
770 lldb_private::Process::HandlePrivateEvent(std::__1::shared_ptr<lldb_private::Event>&) + 308 (LLDB + 3328216) [0x112f488d8]
770 lldb_private::Process::ShouldBroadcastEvent(lldb_private::Event*) + 360 (LLDB + 3343432) [0x112f4c448]
770 lldb_private::ThreadList::ShouldStop(lldb_private::Event*) + 840 (LLDB + 3681968) [0x112f9eeb0]
770 lldb_private::Thread::ShouldStop(lldb_private::Event*) + 956 (LLDB + 3649332) [0x112f96f34]
770 lldb_private::StopInfoBreakpoint::ShouldStopSynchronous(lldb_private::Event*) + 400 (LLDB + 3502008) [0x112f72fb8]
770 lldb_private::BreakpointSite::ShouldStop(lldb_private::StoppointCallbackContext*) + 136 (LLDB + 1841680) [0x112ddda10]
770 lldb_private::BreakpointLocationCollection::ShouldStop(lldb_private::StoppointCallbackContext*) + 192 (LLDB + 1780960) [0x112dcece0]
770 lldb_private::BreakpointLocation::ShouldStop(lldb_private::StoppointCallbackContext*) + 180 (LLDB + 1777068) [0x112dcddac]
770 DynamicLoaderMacOS::NotifyBreakpointHit(void*, lldb_private::StoppointCallbackContext*, unsigned long long, unsigned long long) + 1192 (LLDB + 4185204) [0x113019c74]
770 DynamicLoaderMacOS::AddBinaries(std::1::vector<unsigned long long, std::1::allocator<unsigned long long> > const&) + 784 (LLDB + 4186680) [0x11301a238]
770 lldb_private::DynamicLoaderDarwin::AddModulesUsingImageInfos(std::__1::vector<lldb_private::DynamicLoaderDarwin::ImageInfo, std::__1::allocator<lldb_private::DynamicLoaderDarwin::ImageInfo> >&) + 1684 (LLDB + 4208280) [0x11301f698]
770 lldb_private::Target::ModulesDidLoad(lldb_private::ModuleList&) + 380 (LLDB + 3554444) [0x112f7fc8c]
770 lldb_private::BreakpointList::UpdateBreakpoints(lldb_private::ModuleList&, bool, bool) + 72 (LLDB + 1770684) [0x112dcc4bc]
770 lldb_private::Breakpoint::ModulesChanged(lldb_private::ModuleList&, bool, bool) + 904 (LLDB + 1743524) [0x112dc5aa4]
770 lldb_private::Breakpoint::ResolveBreakpointInModules(lldb_private::ModuleList&, bool) + 224 (LLDB + 1742220) [0x112dc558c]
770 lldb_private::SearchFilter::SearchInModuleList(lldb_private::Searcher&, lldb_private::ModuleList&) + 344 (LLDB + 2229540) [0x112e3c524]
770 lldb_private::SearchFilter::DoModuleIteration(lldb_private::SymbolContext const&, lldb_private::Searcher&) + 144 (LLDB + 2228692) [0x112e3c1d4]
770 lldb_private::BreakpointResolverFileLine::SearchCallback(lldb_private::SearchFilter&, lldb_private::SymbolContext&, lldb_private::Address*) + 248 (LLDB + 1824776) [0x112dd9808]
770 lldb_private::Module::GetCompileUnitAtIndex(unsigned long) + 104 (LLDB + 2136864) [0x112e25b20]
770 lldb_private::SymbolFile::GetCompileUnitAtIndex(unsigned int) + 200 (LLDB + 3085300) [0x112f0d3f4]
770 SymbolFileDWARFDebugMap::ParseCompileUnitAtIndex(unsigned int) + 80 (LLDB + 6835456) [0x1132a0d00]
770 SymbolFileDWARFDebugMap::GetModuleByCompUnitInfo(SymbolFileDWARFDebugMap::CompileUnitInfo*) + 1248 (LLDB + 6830824) [0x11329fae8]
770 std::__1::shared_ptr<DebugMapModule> std::__1::allocate_shared<DebugMapModule, std::1::allocator<DebugMapModule>, std::1::shared_ptr<lldb_private::Module>, unsigned int, lldb_private::FileSpec&, lldb_private::ArchSpec&, lldb_private::ConstString*, int, std::__1::chrono::time_point<std::__1::chrono::system_clock, std::1::chrono::duration<long long, std::1::ratio<1l, 1000000000l> > >, void>(std::1::allocator<DebugMapModule> const&, std::1::shared_ptr<lldb_private::Module>&&, unsigned int&&, lldb_private::FileSpec&, lldb_private::ArchSpec&, lldb_private::ConstString*&&, int&&, std::__1::chrono::time_point<std::__1::chrono::system_clock, std::1::chrono::duration<long long, std::1::ratio<1l, 1000000000l> > >&&) + 140 (LLDB + 6852460) [0x1132a4f6c]
770 lldb_private::Module::Module(lldb_private::FileSpec const&, lldb_private::ArchSpec const&, lldb_private::ConstString const*, unsigned long long, std::__1::chrono::time_point<std::__1::chrono::system_clock, std::1::chrono::duration<long long, std::1::ratio<1l, 1000000000l> > > const&) + 336 (LLDB + 2132876) [0x112e24b8c]
770 std::__1::recursive_mutex::lock() + 16 (__threading_support:283 in libc++.1.dylib + 78548) [0x19ae1e2d4]
770 _pthread_mutex_firstfit_lock_slow + 248 (pthread_mutex.c:1490 in libsystem_pthread.dylib + 6812) [0x19aeeca9c]
770 __psynch_mutexwait + 8 (libsystem_kernel.dylib + 15004) [0x19aeb5a9c]
*770 psynch_mtxcontinue + 0 (kern_synch.c:785 in com.apple.kec.pthread + 8060) [0xfffffe000b861b9c] (blocked by turnstile waiting for lldb-rpc-server [12692] [unique pid 112195] thread 0x2c4a6c)
And the deadlock is caused by DebuggerStats::ReportStatistics:
Thread 0x2c4a6c Thread name "RPC packet thread for client tid 002c4a6b (2902635)" 770 samples (1-770) priority 31 (base 31)
770 thread_start + 8 (libsystem_pthread.dylib + 7724) [0x19aeece2c]
770 _pthread_start + 148 (pthread.c:893 in libsystem_pthread.dylib + 28780) [0x19aef206c]
770 Packets::RunReadThread(void*) + 12 (RPCCommon.cpp:1401 in lldb-rpc-server + 401672) [0x10239a108]
770 Packets::ReadThread() + 276 (RPCCommon.cpp:1426 in lldb-rpc-server + 401960) [0x10239a228]
770 Packets::ProcessPackets() + 528 (RPCCommon.cpp:1568 in lldb-rpc-server + 402628) [0x10239a4c4]
770 rpc_common::Connection::HandleRPCPacket(rpc_common::RPCPacket&) + 72 (RPCCommon.cpp:578 in lldb-rpc-server + 390928) [0x102397710]
770 rpc_common::Connection::PrivateHandleRPCPacket(rpc_common::RPCPacket&, rpc_common::RPCPacket&, bool&) + 628 (RPCCommon.cpp:400 in lldb-rpc-server + 387012) [0x1023967c4]
770 rpc_server::_ZN4lldb8SBTarget13GetStatisticsEv::HandleRPCCall(rpc_common::Connection&, rpc_common::RPCStream&, rpc_common::RPCStream&) + 56 (RPCUserServer_SBTarget.cpp:144 in lldb-rpc-server + 343236) [0x10238bcc4]
770 lldb::SBTarget::GetStatistics() + 252 (LLDB + 642604) [0x112cb8e2c]
770 lldb_private::DebuggerStats::ReportStatistics(lldb_private::Debugger&, lldb_private::Target*) + 968 (LLDB + 3493400) [0x112f70e18]
770 lldb_private::SymbolFile::GetSymtab() + 44 (LLDB + 3085784) [0x112f0d5d8]
770 std::__1::recursive_mutex::lock() + 16 (__threading_support:283 in libc++.1.dylib + 78548) [0x19ae1e2d4]
770 _pthread_mutex_firstfit_lock_slow + 248 (pthread_mutex.c:1490 in libsystem_pthread.dylib + 6812) [0x19aeeca9c]
770 __psynch_mutexwait + 8 (libsystem_kernel.dylib + 15004) [0x19aeb5a9c]
*770 psynch_mtxcontinue + 0 (kern_synch.c:785 in com.apple.kec.pthread + 8060) [0xfffffe000b861b9c] (blocked by turnstile waiting for lldb-rpc-server [12692] [unique pid 112195] thread 0x2c4a28)
</pre>
<img width="1px" height="1px" alt="" src="http://email.email.llvm.org/o/eJztWltv2zoS_jXOCxFDou4PeUicdlug7RbJKfZgXwxKom02sqQjUkn9788MKdmSLNtKWmDRg01Tx6LImW--IYfDS1yku5t7HtfrNa8eFVNy5tzC7wMvi0phgZBKJJIo9sThc8NJViRPpOKqrnKekniHDaz7mdV8fi7SOuNGyr-4us2gPkgpcvNiUWQZT_RzrfiPbkuWp2RTZKkkQpGXjcg4fhGKV0xp3VVRrzeEZZnGwX8oliuy1WIluYbCWhKW_FWLSuRrwlmyIcUK626JyAkCnhNyV4PMFWF5AS-qpjkRklRivVG6IgjfijSFYmieVBzggzyhJM9WM7pAVFlRPEmSiSeNMWE54YC-LsmGPWNllkGzdKdpy7EpMeaTrbYahKC1oAnUVTtsoQqy5korv8AZGPHHhikiN0WdpfmMBgrUliXIWhUVkWJbgiLTVKKquDZy1UtxnfFnnjVGFTka2BKoUKaRI4F-kFZLBHb_n9uH97qLfGYlkbttXGSkzOo1UAW-KF5Mt0iKXDGRY4uGU7Ao5uRZSBGbp6NOBViQbu1WocAQSeAF-LfxQlkVCZdSo2RPKDrnL0TW8XULGg3GmvOCrKDDyD03CqBBb1gDJKmONPcccASraQzu0I5H-V3vaaVFzpvOJTnC7UCNObbSFMMAaZGiOrC32qkNvl5DH4WeDqwwID1T8-5I-MArbtgotpxIph0KqBLoc6uq2EIbCcrQgaAahgr0dBArSQqdTo_PFwZQYsmrZ55qThDmC1KLPRfaZ0Ip7OApDGGesFq25sDQXoH6POEaXZal8XVVJtdaVoXEYUfR6L7XQC0jLxV2moowGJ-GU_Lp0_0dIGVb_lJUT6Zr_ZkUKYc-BU7q2orgcKwQ6wdNXEZDQvZFOQggM0pnzgJxzBuS5yKHoJCz7Bo4UcBUWIp05tzbNKDWjEYz5x00QjlBYDX0SZAT2tdQABXwVVmJohJqRxwbX8UMGHBsbN3AwrYYdADIEvRUCqrdkRArZyKWO6n4dlmaCvN0B2W6QhBQF1XMvDvrhx0xzhNOk5l338ptJLdNO7JtV0tvZSYQQcPIQRefU0jDILR6GlfU8vsaG51I4hIMf0bWdIT-UEhlyP7Mkn8__mlKTclC9-A_wItlkYkceX4ugGh6q7WBaociXu1sfPSp51Lq-Acwth8kth0NwZyF8wXG4TM3EO7AK6-DZFtWDxP1qRV4XUiUR3HquUeQGlBSpUajbf6s6lyHXvO0XOJz0x_72L-azmkeHtGpX81rjCyNPdjRiiLTfRSrzai79HFM7bUul41eZiaBovrFynBsLEjDGlLVDBhdpShxsgWdplh_GhrdHquOEzihZXVZXXkO4-EIqyed3bPhoc7PWWB8a4fWAIbrB2HYg-Gmtj_m3KlAPkCgzniD5R1MUBDrwmP3yA2reArDcdw_piFy7e8HixUOwNOQ2n4ffBim4xxOhf-ok4K7qmBpwqRqDTiFsDOW_SNu4R_tw0tc9zS8kwCNMz_B7NrF-KiKchKy0B0g80M78vtejziPrTPILmB7E67I8we43MgZMBb5K-dUb7yADHF8zFfFHQB8KguRH7H3uMsTSIrzopaTALuD2Oh4FrWsPpEBXcXnXHwW8gHqo9ibcZ5UfKFbLCDgxZDhLCCPhNS-G9KdPs926Np-2As-aZoy-7z_JwL_1GTeh5z7l5kR9WdLG6btyO-bkUCyMMWMVxjyy-APYq8dAIT-KEwTcMPxXH_GgPsdpHgi-QTRilc6AzF4vhRKrHYHYz7g6uAw1S9ODJeTJixInUuxxgVrVkD2jR-nStu5ZuAu1w49anVSO9uxILMJLo3uiRbfpumdyFklcMkWDpOQZ-iKOhMYN2NYvZs7jLRw3hH8Dys2iMiHCSoI3aHFfn-cgcWMOpfDw8Vu2mPhnlUvIt_T0Cxcv-Hi8-OWrTkGQTk6Ce9peYWGvci3pF0TpBpyO7RCrBrwSq2QDnld-dFreD09n7EK1q3me8PkvUgR8OjAN1X01NzJVIZplue58NOfJ1ZJOHWoX4DciVn7FOFbmcL7w5vxCW6AfkFMttj-bbr1MOxi2HL7cStx49caM8GcnhcWG5av-SQvjNsRWe7AENfx6MAQj7HpAenVpjxwWWTPHbd8zPfbTNPNalYU9MgcSulgOvS8V3eyswY9clYlm_ciU7zqlrR2GKxjE6Suhvsf_vj0Mz6Q3L6NYGDkuT0buZOgE99i4yusvC8Mvo96MxWzglEj9d5eM292pofR6bZLSBPojq0N_YgOrLXTt1o7oZs2HbQC28EXeQ9rmxec8W9D2Umbu_ycrAVTWIXLsUPqRN3-ys8OqRsEfj-BjULrdRPAJGoGu_GLYlsCNd9yoW5h8Ka4FR32MoTDPsrAmZCFh34v2nDqxXRatvpK1IZodOJE5BihWrKHixwr9JzBVoWVOqu398IO9APQ3i65Qf2VVZJPxj2Yc_3Q8dzevpVDmZVaP8f3NOBAt-k3dzuEj9hNwhSebdYx1VQ_rB2GIwCsgxyol0zTaHVq_-gN9l3YsGmBt8Pj3elMkC9Ny-NWFxLvIx1jDS5sI_Xa9nvNWOhBxzyWPDkdmyDIna2wwKD_qPAArVk4NcqO-TFbD-a7Elu-bBKFxfmqzT52gscUY4x066Z1M1sBN-fWO7oWVLJ10mRb7U_WLnW6u5749WiBddZxvanw7R70Gwl9P_onffHrvdnq6in-B3jW72Uhw0AK-VVvmwUCqbs6OiD5iWBzftpto8VI1tF6-EKu1fr5QrWOt9uaZ7Y-fnu3H-1fOIN9QkhZaNhPszh14zeuKCZPNBVP6kqKZ740Z_260JATHvYEEOlyaQ7zwGFLWZd4Cg1Vadge-cHAvoPfud09Xgw9t7PpFjFuc_oTKfWoUfuzSW3BciUqqVZCLdGKpcyKl25a26urDy1tN7IunVr6oU17ZvCEjZwT_qwhy1LiJrkB98LE6CHuE69ynnXA2Z7V3ekDdLH3a9GBmluNsMWnfuAFCpHXXCPQMQxxLfV7TSu4HllNiu2clWXG5088mTfMNilkJ86t8IfDmIlD3441eH3OjT7U93b0hRi8bsEJEoNRAy82DA78UZhNcRmH7b27Ohd_AcZSpATGkx15WK66J_gQWdtD9O7nbXM1YH9FQUii7x1oLBOuIGHp6XsDfoKsDu4NPHxdkBKWfPpejX6DFiaZ4DmUgAmWpdui10MaWdR3PH3u-f_LA8eXB75qIg_HtQ8gaX9MOzyF13tuQD-sCbZFPk_KUscFy9aAhl1Mn03ZftCJCBZ1ImaPLIlHsHSB7JeBgT8OgfpnIPTOYzQESsfWJMcgmkPYtuwAxKPhKBDPD08DoT4NB0DcZDzIayggYJlo8ftUIO-eXplDbcBg8AGeYYvOu8OBwKgXveAkcCeyoiHwIDh5MDcJenMS_1oLFkeCh-_NZuTeWH_cT3hmesraMLDsQZf1gxNu6tnb3KUyecN_v7goPXy8M3v3tgPr70PYe_c8cCHuY42Y3-HtpP2QHHK2nfS-6bztIPoGkB816mWLcz-i3JMEuU7vNhAQFMbJGYI6-bTB1OqatfsS3St8h7HuDe4hwSDvndTRJA6HcfSk5sGJz4SJaSy1b9udzNYbwzrXCfzB_RA3ctzBxlVgcfvSHsn0PTUoUizuEDnYwMWts6B_VLKyUu_MDZUeht8nNf4Nkt7_WTr7z01UzR03zcAVv7F937N9L_Cjq_TGSSOYcq-UUBmfdD0eb4CbW7Qmt80Bkb6f26a7Ei-dV3XeXGxXevRLwqr9LWe0C3yGy3RMYfEGhDBXOK7qKrvZKFVq7fQ9_K6F2tTxHCiGhyx7bv9cg7DvMA3Ao5CyxoO5914IUfhqcxNEMXOidOWnUWKtbOZBtpPGaRKnMEtHnnOVsZhn8gbIgwwYb1prEXj_1ru_EjfUohSW4oEdWtSN5owxmO2Zx6jr2bGdzFyLb5nI5ohjXlTrq-pGQwL2JLzMgCp5eMmk3pXgjbokY7sYyhttrFaborr5LrZAy4ZtrzSUG23K369AAr8">