<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:"Segoe UI";
panose-1:2 11 5 2 4 2 4 2 2 3;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:purple;
text-decoration:underline;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0in;
margin-right:0in;
margin-bottom:0in;
margin-left:.5in;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
p.msonormal0, li.msonormal0, div.msonormal0
{mso-style-name:msonormal;
mso-margin-top-alt:auto;
margin-right:0in;
mso-margin-bottom-alt:auto;
margin-left:0in;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
span.EmailStyle19
{mso-style-type:personal;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.EmailStyle20
{mso-style-type:personal;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.EmailStyle21
{mso-style-type:personal;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.EmailStyle22
{mso-style-type:personal;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.EmailStyle23
{mso-style-type:personal;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.EmailStyle25
{mso-style-type:personal-reply;
font-family:"Calibri",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:104034890;
mso-list-template-ids:-2136460368;}
@list l0:level1
{mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level2
{mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level3
{mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level4
{mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level5
{mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level6
{mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level7
{mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level8
{mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level9
{mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l1
{mso-list-id:611401726;
mso-list-template-ids:1680621638;}
@list l1:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level2
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level5
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level8
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2
{mso-list-id:771632594;
mso-list-template-ids:846612018;}
@list l3
{mso-list-id:1214390884;
mso-list-template-ids:1098008072;}
@list l3:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level2
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level5
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level8
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
ol
{margin-bottom:0in;}
ul
{margin-bottom:0in;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="EN-US" link="blue" vlink="purple">
<div class="WordSection1">
<p class="MsoNormal">Closing the gap on this. Moving nGraph MLIR pointer to the 3c5ddd58 git hash did indeed fix the problem. Thanks so much for all the help.<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal"><b>From:</b> Straw, Adam D <br>
<b>Sent:</b> Wednesday, May 27, 2020 11:54 AM<br>
<b>To:</b> Lang Hames <lhames@gmail.com>; David Blaikie <dblaikie@gmail.com><br>
<b>Cc:</b> llvm-dev@lists.llvm.org; Cyphers, Scott <Scott.Cyphers@intel.com>; Bobba, Jayaram <jayaram.bobba@intel.com>; Mostafa, Nagy H <Nagy.H.Mostafa@intel.com>; Iyer, Swapna <swapna.iyer@intel.com>; Caballero, Diego <diego.caballero@intel.com><br>
<b>Subject:</b> RE: [llvm-dev] [ORC JIT][MLIR] GDBRegistrationListener "second attempt to perform debug registration" assert<o:p></o:p></p>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">Lang,<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">It looks like the change I need is here, correct?<o:p></o:p></p>
<p class="MsoNormal"><span style="font-size:10.5pt;font-family:"Segoe UI",sans-serif"><a href="https://github.com/llvm/llvm-project/commit/3c5dd5863c34ecd51e9d2a49929877d8151dea39" target="_blank" title="https://github.com/llvm/llvm-project/commit/3c5dd5863c34ecd51e9d2a49929877d8151dea39">https://github.com/llvm/llvm-project/commit/3c5dd5863c34ecd51e9d2a49929877d8151dea39</a><o:p></o:p></span></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">I would like to move the nGraph MLIR pointer all the way to the tip of master but it is expensive (lots of compile and link issues) so hoping to take a baby step.<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">Adam <o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal"><b>From:</b> Straw, Adam D <br>
<b>Sent:</b> Tuesday, May 26, 2020 1:17 PM<br>
<b>To:</b> Lang Hames <<a href="mailto:lhames@gmail.com">lhames@gmail.com</a>>; David Blaikie <<a href="mailto:dblaikie@gmail.com">dblaikie@gmail.com</a>><br>
<b>Cc:</b> <a href="mailto:llvm-dev@lists.llvm.org">llvm-dev@lists.llvm.org</a>; Cyphers, Scott <<a href="mailto:Scott.Cyphers@intel.com">Scott.Cyphers@intel.com</a>>; Bobba, Jayaram <<a href="mailto:jayaram.bobba@intel.com">jayaram.bobba@intel.com</a>>; Mostafa,
Nagy H <<a href="mailto:Nagy.H.Mostafa@intel.com">Nagy.H.Mostafa@intel.com</a>>; Iyer, Swapna <<a href="mailto:swapna.iyer@intel.com">swapna.iyer@intel.com</a>><br>
<b>Subject:</b> RE: [llvm-dev] [ORC JIT][MLIR] GDBRegistrationListener "second attempt to perform debug registration" assert<o:p></o:p></p>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">Referring to the log messages from my previous mail… I confused myself (and probably others) by reading the “<span style="font-family:"Courier New"">Adding MemMgr 0x55555959f440</span>“ message as “<b><i><span style="font-family:"Courier New"">Registering</span></i></b><span style="font-family:"Courier New"">
MemMgr 0x55555959f440</span>”. Thus the address mismatch made no sense. How could we be registering a `MemMgr` address/key that does not match once we arrive in `notifyObjectLoaded` method?<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">Answer: Because the registrations is NOT coming from RTDyldObjectLinkingLayer.cpp. I found this first by adding an additional debug statement (highlighted) thinking the `reinterpret_cast` may be at fault for the key mismatch. I never
saw this print and I then realized my “Adding” vs. “Registering” mistake.<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> // Run EventListener notifyLoaded callbacks.<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> {<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> std::lock_guard<std::mutex> Lock(RTDyldLayerMutex);<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> dbgs() << "Registering MemMgr " << MemMgr << "\n";<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> auto LOIItr = LoadedObjInfos.find(MemMgr);<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> assert(LOIItr != LoadedObjInfos.end() && "LoadedObjInfo missing");<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> for (auto *L : EventListeners) {<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New"">
<span style="background:yellow;mso-highlight:yellow">dbgs() << " key = " << reinterpret_cast<uintptr_t>(MemMgr) << "\n";</span><o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> L->notifyObjectLoaded(<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> static_cast<uint64_t>(reinterpret_cast<uintptr_t>(MemMgr)), *Obj,<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> *LOIItr->second);<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> }<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> LoadedObjInfos.erase(MemMgr);<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> }</span><o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">So, where is the registration coming from? Remember that I am on a slightly older version of MLIR. My mlir/lib/ExecutionEngine/ExecutionEngine.cpp has the following code with highlighted debug statement added by me:<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> objectLayer->setNotifyLoaded(<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> [engine = engine.get()](<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> llvm::orc::VModuleKey, const llvm::object::ObjectFile &object,<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> const llvm::RuntimeDyld::LoadedObjectInfo &objectInfo) {<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> if (engine->gdbListener) {<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> uint64_t key = static_cast<uint64_t>(<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> reinterpret_cast<uintptr_t>(object.getData().data()));<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New"">
<span style="background:yellow;mso-highlight:yellow">dbgs() << "ExecutionEngine key = " << key << "\n";</span><o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> engine->gdbListener->notifyObjectLoaded(key, object, objectInfo);<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> }<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""> });<o:p></o:p></span></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">The key printed here matches the key printed in `notifyObjectLoaded`. This code is NOT present at the tip of MLIR master. Hence the solution here may be to move to latest MLIR master which I probably should have tried before sending this
mail. Will do that now.<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">Adam<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal"><b>From:</b> Straw, Adam D <br>
<b>Sent:</b> Tuesday, May 26, 2020 9:41 AM<br>
<b>To:</b> Lang Hames <<a href="mailto:lhames@gmail.com">lhames@gmail.com</a>>; David Blaikie <<a href="mailto:dblaikie@gmail.com">dblaikie@gmail.com</a>><br>
<b>Cc:</b> <a href="mailto:llvm-dev@lists.llvm.org">llvm-dev@lists.llvm.org</a>; Cyphers, Scott <<a href="mailto:Scott.Cyphers@intel.com">Scott.Cyphers@intel.com</a>>; Bobba, Jayaram <<a href="mailto:jayaram.bobba@intel.com">jayaram.bobba@intel.com</a>>; Mostafa,
Nagy H <<a href="mailto:Nagy.H.Mostafa@intel.com">Nagy.H.Mostafa@intel.com</a>>; Iyer, Swapna <<a href="mailto:swapna.iyer@intel.com">swapna.iyer@intel.com</a>><br>
<b>Subject:</b> RE: [llvm-dev] [ORC JIT][MLIR] GDBRegistrationListener "second attempt to perform debug registration" assert<o:p></o:p></p>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">Sorry for the delay. Hope everyone had a good long weekend. See attached log captured from GDB session. Here is the punchline:<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New";background:yellow;mso-highlight:yellow">Adding MemMgr 0x55555959f440</span><span style="font-family:"Courier New""><o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New"">unit-test: /localdisk/adstraw/ngraph/build/mlir_project/llvm-project/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp:164: virtual void {anonymous}::GDBJITRegistrationListener::notifyObjectLoaded(llvm::JITEventListener::ObjectKey,
const llvm::object::ObjectFile&, const llvm::RuntimeDyld::LoadedObjectInfo&): Assertion `ObjectBufferMap.find(K) == ObjectBufferMap.end() && "Second attempt to perform debug registration."' failed.<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New""><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New"">(gdb) print/x K<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-family:"Courier New"">$2 = 0x555559605c50</span><o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">The key (K) causing the assert a) does not match the addresses of the most recently added MemMgr and b) does not match the address of ANY MemMgr in the entire log file. The plot thickens.<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">Wondering if there is some other object (besides a MemMgr) that registers its address-as-key into `ObjectBufferMap`.
<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">Adam <o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal"><b>From:</b> Lang Hames <<a href="mailto:lhames@gmail.com">lhames@gmail.com</a>>
<br>
<b>Sent:</b> Thursday, May 21, 2020 4:53 PM<br>
<b>To:</b> David Blaikie <<a href="mailto:dblaikie@gmail.com">dblaikie@gmail.com</a>><br>
<b>Cc:</b> Straw, Adam D <<a href="mailto:adam.d.straw@intel.com">adam.d.straw@intel.com</a>>;
<a href="mailto:llvm-dev@lists.llvm.org">llvm-dev@lists.llvm.org</a>; Cyphers, Scott <<a href="mailto:scott.cyphers@intel.com">scott.cyphers@intel.com</a>>; Bobba, Jayaram <<a href="mailto:jayaram.bobba@intel.com">jayaram.bobba@intel.com</a>>; Mostafa, Nagy
H <<a href="mailto:nagy.h.mostafa@intel.com">nagy.h.mostafa@intel.com</a>><br>
<b>Subject:</b> Re: [llvm-dev] [ORC JIT][MLIR] GDBRegistrationListener "second attempt to perform debug registration" assert<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<div>
<div>
<p class="MsoNormal">Hi Adam,<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal">Calls to the listeners should be protected by the RTDyldLayerMutex. Could you apply the attached patch and share the debugging output from one of the failing runs?<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal">Regards,<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal">Lang.<o:p></o:p></p>
</div>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<div>
<p class="MsoNormal">On Wed, May 20, 2020 at 8:00 PM David Blaikie <<a href="mailto:dblaikie@gmail.com">dblaikie@gmail.com</a>> wrote:<o:p></o:p></p>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt">
<div>
<p class="MsoNormal">+Lang<o:p></o:p></p>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<div>
<p class="MsoNormal">On Wed, May 20, 2020 at 4:44 PM Straw, Adam D via llvm-dev <<a href="mailto:llvm-dev@lists.llvm.org" target="_blank">llvm-dev@lists.llvm.org</a>> wrote:<o:p></o:p></p>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt">
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Hi all,<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Attention: Lang Hames<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">I am developing the
<a href="https://github.com/NervanaSystems/ngraph/tree/master/src/contrib/mlir" target="_blank">
nGraph MLIR</a> implementation and hitting the following assert while running nGraph unit tests:<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-family:"Courier New""> assert(ObjectBufferMap.find(K) == ObjectBufferMap.end() &&</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-family:"Courier New""> "Second attempt to perform debug registration.");</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Here is a
<a href="https://github.com/llvm/llvm-project/blob/3d5360a4398bfa6878f94ca9ac55bc568692c765/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp#L163" target="_blank">
permalink</a> for that line of code on GitHub. The failure only occurs when running multiple unit tests back-to-back. Inevitably, if I rerun the failing unit test, it passes. The failure also tends to move around with a different unit tests failing on successive
runs.<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">I am able to hit the failure in GDB. Here is a partial backtrace:<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New"">#3 0x00007fffec4cf412 in __GI___assert_fail (assertion=assertion@entry=0x7ffff1d145e0 "ObjectBufferMap.find(K) == ObjectBufferMap.end() && \"Second attempt to perform debug registration.\"",
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New""> file=file@entry=0x7ffff1d144d8 "/localdisk/adstraw/ngraph/build/mlir_project/llvm-project/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp", line=line@entry=164,
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New""> function=function@entry=0x7ffff1d15520 <(anonymous namespace)::GDBJITRegistrationListener::notifyObjectLoaded(unsigned long, llvm::object::ObjectFile const&, llvm::RuntimeDyld::LoadedObjectInfo const&)::__PRETTY_FUNCTION__>
"virtual void {anonymous}::GDBJITRegistrationListener::notifyObjectLoaded(llvm::JITEventListener::ObjectKey, const llvm::object::ObjectFile&, const llvm::RuntimeDyld::LoadedObjectInfo&)")</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New""> at assert.c:101</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New"">#4 0x00007ffff01ed4d3 in (anonymous namespace)::GDBJITRegistrationListener::notifyObjectLoaded (this=0x5555591a9270,
<span style="color:black;background:yellow">K=93825060893584</span>, Obj=..., L=...)</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New""> at /localdisk/adstraw/ngraph/build/mlir_project/llvm-project/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp:163</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New"">#5 0x00007ffff01b0eca in std::function<void (unsigned long, llvm::object::ObjectFile const&, llvm::RuntimeDyld::LoadedObjectInfo const&)>::operator()(unsigned long, llvm::object::ObjectFile const&, llvm::RuntimeDyld::LoadedObjectInfo
const&) const (__args#2=..., __args#1=..., __args#0=<optimized out>, this=0x5555594e09b8) at /usr/include/c++/7/bits/std_function.h:706</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New"">#6 llvm::orc::RTDyldObjectLinkingLayer::onObjLoad (this=<optimized out>,
<span style="color:black;background:yellow">K=3</span>, R=..., Obj=..., MemMgr=0x5555591c4160, LoadedObjInfo=std::unique_ptr<llvm::RuntimeDyld::LoadedObjectInfo> = {...},
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New""> Resolved=std::map with 8 elements = {...}, InternalSymbols=std::set with 0 elements)</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New""> at /localdisk/adstraw/ngraph/build/mlir_project/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp:267</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">A little debug context:<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<ul type="disc">
<li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l3 level1 lfo3">
<b>Frame #6</b> with K=3 we are about to execute the `NotifyLoaded` callback<o:p></o:p></li><li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l3 level1 lfo3">
<b>Frame #5</b> we execute the `NotifyLoaded` callback which apparently transforms K from 3 to 93825060893584 (0x5555596CF390) which looks like a pointer. I don’t know the code well enough to grok what’s going on here. I’m not even sure where this callback
resides, but it could be <a href="https://github.com/llvm/llvm-project/blob/c66f89005f6d23b6885d8f93f33ff27dc60ce7dd/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp#L312" target="_blank">
here</a> in RTDyldObjectLinkingLater.cpp where we are using the address of the `MemMgr` as K.<o:p></o:p></li><li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l3 level1 lfo3">
<b>Frame #4 </b>now in `notifyObjectLoaded` with K=93825060893584 which will go on to assert in
<b>frame #3</b> due to the fact that that key (K) already exists in the map<o:p></o:p></li></ul>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">All of this leads me to believe there is some sort of race. Theory: with two unit tests A and B running back to back…<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<ol start="1" type="1">
<li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l0 level1 lfo6">
Unit test A allocates and deallocates a `MemMgr` (or whatever other object we are using for a key) at address X<o:p></o:p></li><li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l0 level1 lfo6">
Unit test A calls ` notifyFreeingObject` to free the object at K=X but gets stuck (does not lock) the ` JITDebugLock` mutex
<a href="https://github.com/llvm/llvm-project/blob/3d5360a4398bfa6878f94ca9ac55bc568692c765/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp#L181" target="_blank">
here</a> for whatever reason<o:p></o:p></li><li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l0 level1 lfo6">
Meanwhile unit test B allocates a `MemMgr` (or whatever other object we are using for a key) at newly freed address X<o:p></o:p></li><li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l0 level1 lfo6">
Unit test B calls `notifyObjectLoaded` to register the object at K=X and locks the ` JITDebugLock`
<a href="https://github.com/llvm/llvm-project/blob/3d5360a4398bfa6878f94ca9ac55bc568692c765/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp#L162" target="_blank">
here</a> with unit test A still waiting in step #2 above<o:p></o:p></li><li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l0 level1 lfo6">
This goes on to assert as the key was not erased (step #2) before it was added (step #4)<o:p></o:p></li></ol>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">I could use a little help to debug the error further. Curiously, we do hit this issue in continuous integration (CI) --- only when running on development systems “by hand”. I
am investigating the differences (perhaps single vs. multi threaded?) as to why this might be the case.<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Note that I am using a slightly old version of MLIR master, I’m at this commit:<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New"">commit 2f8b4545f4960778e37114c024073d208751ca89</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New"">Author: Adam Straw <<a href="mailto:adam.d.straw@intel.com" target="_blank">adam.d.straw@intel.com</a>></span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New"">Date: Tue Apr 14 22:49:18 2020 +0300</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New""> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New""> [mlir] Fix assert on signed integer type in EDSC</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New""> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New""> Integer type in Std dialect is signless so we should be checking</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New""> for signless integer type instead of signed integer type in EDSC.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New""> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in">
<span style="font-family:"Courier New""> Differential Revision: <a href="https://reviews.llvm.org/D78144" target="_blank">
https://reviews.llvm.org/D78144</a></span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Thanks in advance for the help.<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Adam<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Note: I am out of office this Fri and the following Mon but otherwise should be prompt with email replies.<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-family:"Courier New""> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
</div>
</div>
<p class="MsoNormal">_______________________________________________<br>
LLVM Developers mailing list<br>
<a href="mailto:llvm-dev@lists.llvm.org" target="_blank">llvm-dev@lists.llvm.org</a><br>
<a href="https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-dev" target="_blank">https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-dev</a><o:p></o:p></p>
</blockquote>
</div>
</blockquote>
</div>
</div>
</body>
</html>