[llvm] OrcV2: also set COFF flag overrides when custom linking layer is used (PR #129533)

[Wenzel Jakob via llvm-commits]

https://github.com/wjakob updated https://github.com/llvm/llvm-project/pull/129533

>From a28bf0a50bd23fc194dee8165e1d7024f9123e5b Mon Sep 17 00:00:00 2001
From: Wenzel Jakob <wenzel.jakob at epfl.ch>
Date: Mon, 3 Mar 2025 22:45:39 +0900
Subject: [PATCH] OrcV2: also set COFF flag overrides when custom linking layer
 is used

---
 llvm/lib/ExecutionEngine/Orc/LLJIT.cpp | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index 972c24abc7506..fe2fa8dd48070 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -950,8 +950,18 @@ Expected<std::unique_ptr<ObjectLayer>>
 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
 
   // If the config state provided an ObjectLinkingLayer factory then use it.
-  if (S.CreateObjectLinkingLayer)
-    return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
+  if (S.CreateObjectLinkingLayer) {
+    auto Layer = S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
+    if (Layer && S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
+      auto *RTDyldLayer =
+          dyn_cast_or_null<RTDyldObjectLinkingLayer>(Layer.get().get());
+      if (RTDyldLayer) {
+        RTDyldLayer->setOverrideObjectFlagsWithResponsibilityFlags(true);
+        RTDyldLayer->setAutoClaimResponsibilityForObjectSymbols(true);
+      }
+    }
+    return Layer;
+  }
 
   // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
   // a new SectionMemoryManager for each object.