[llvm] OrcV2: also set COFF flag overrides when custom linking layer is used (PR #129533)

[Wenzel Jakob via llvm-commits]

https://github.com/wjakob updated https://github.com/llvm/llvm-project/pull/129533

>From a28bf0a50bd23fc194dee8165e1d7024f9123e5b Mon Sep 17 00:00:00 2001
From: Wenzel Jakob <wenzel.jakob at epfl.ch>
Date: Mon, 3 Mar 2025 22:45:39 +0900
Subject: [PATCH 1/3] OrcV2: also set COFF flag overrides when custom linking
 layer is used

---
 llvm/lib/ExecutionEngine/Orc/LLJIT.cpp | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index 972c24abc7506..fe2fa8dd48070 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -950,8 +950,18 @@ Expected<std::unique_ptr<ObjectLayer>>
 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
 
   // If the config state provided an ObjectLinkingLayer factory then use it.
-  if (S.CreateObjectLinkingLayer)
-    return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
+  if (S.CreateObjectLinkingLayer) {
+    auto Layer = S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
+    if (Layer && S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
+      auto *RTDyldLayer =
+          dyn_cast_or_null<RTDyldObjectLinkingLayer>(Layer.get().get());
+      if (RTDyldLayer) {
+        RTDyldLayer->setOverrideObjectFlagsWithResponsibilityFlags(true);
+        RTDyldLayer->setAutoClaimResponsibilityForObjectSymbols(true);
+      }
+    }
+    return Layer;
+  }
 
   // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
   // a new SectionMemoryManager for each object.

>From eecf6af31d9dfac8502bdfe30052cf3d96637a0c Mon Sep 17 00:00:00 2001
From: Wenzel Jakob <wenzel.jakob at epfl.ch>
Date: Mon, 10 Mar 2025 15:23:22 +0900
Subject: [PATCH 2/3] incorporate feedback

---
 llvm/lib/ExecutionEngine/Orc/LLJIT.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index de339c901c021..6d21993f4323e 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -951,10 +951,9 @@ LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
   // If the config state provided an ObjectLinkingLayer factory then use it.
   if (S.CreateObjectLinkingLayer) {
     auto Layer = S.CreateObjectLinkingLayer(ES);
-    if (Layer && S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
-      auto *RTDyldLayer =
-          dyn_cast_or_null<RTDyldObjectLinkingLayer>(Layer.get().get());
-      if (RTDyldLayer) {
+    if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
+      if (auto *RTDyldLayer =
+              dyn_cast<RTDyldObjectLinkingLayer>(Layer->get())) {
         RTDyldLayer->setOverrideObjectFlagsWithResponsibilityFlags(true);
         RTDyldLayer->setAutoClaimResponsibilityForObjectSymbols(true);
       }

>From 10b43e122ce77299f8c78d862aef34988dae3e0f Mon Sep 17 00:00:00 2001
From: Wenzel Jakob <wjakob at nvidia.com>
Date: Tue, 11 Mar 2025 23:48:09 +0900
Subject: [PATCH 3/3] Revert "incorporate feedback"

This reverts commit eecf6af31d9dfac8502bdfe30052cf3d96637a0c.
---
 llvm/lib/ExecutionEngine/Orc/LLJIT.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index 6d21993f4323e..de339c901c021 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -951,9 +951,10 @@ LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
   // If the config state provided an ObjectLinkingLayer factory then use it.
   if (S.CreateObjectLinkingLayer) {
     auto Layer = S.CreateObjectLinkingLayer(ES);
-    if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
-      if (auto *RTDyldLayer =
-              dyn_cast<RTDyldObjectLinkingLayer>(Layer->get())) {
+    if (Layer && S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
+      auto *RTDyldLayer =
+          dyn_cast_or_null<RTDyldObjectLinkingLayer>(Layer.get().get());
+      if (RTDyldLayer) {
         RTDyldLayer->setOverrideObjectFlagsWithResponsibilityFlags(true);
         RTDyldLayer->setAutoClaimResponsibilityForObjectSymbols(true);
       }