[llvm] [Offload][InputGenGPU] Add GPU instrumentation pass policy (PR #216417)
Vincent Arcila via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 10:17:35 PDT 2026
https://github.com/jandrovins updated https://github.com/llvm/llvm-project/pull/216417
>From 0fd94ee68387a62370a69e56c8bee44165c18662 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Mon, 27 Jul 2026 16:39:39 -0700
Subject: [PATCH 1/8] [Instrumentor] Add .json option for keeping symbols
visible after linking
---
.../llvm/Transforms/IPO/Instrumentor.h | 28 +++++++++++++++
llvm/lib/Transforms/IPO/Instrumentor.cpp | 20 +++++++++--
.../Transforms/IPO/InstrumentorConfigFile.cpp | 35 +++++++++++++++++++
.../Instrumentor/default_config.json | 6 ++--
4 files changed, 85 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/Transforms/IPO/Instrumentor.h b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
index 5da53ac0ada15..6fbc620a7c4a9 100644
--- a/llvm/include/llvm/Transforms/IPO/Instrumentor.h
+++ b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
@@ -13,10 +13,12 @@
#ifndef LLVM_TRANSFORMS_IPO_INSTRUMENTOR_H
#define LLVM_TRANSFORMS_IPO_INSTRUMENTOR_H
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/EnumeratedArray.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
@@ -273,6 +275,7 @@ struct BaseConfigurationOption {
enum KindTy {
STRING,
BOOLEAN,
+ STRING_LIST,
};
/// Create a boolean option with \p Name name, \p Description description and
@@ -287,6 +290,13 @@ struct BaseConfigurationOption {
createStringOption(InstrumentationConfig &IC, StringRef Name,
StringRef Description, StringRef DefaultValue);
+ /// Create a string-list option with \p Name name, \p Description
+ /// description and \p DefaultValue as string-list default value.
+ LLVM_ABI static std::unique_ptr<BaseConfigurationOption>
+ createStringListOption(InstrumentationConfig &IC, StringRef Name,
+ StringRef Description,
+ ArrayRef<StringRef> DefaultValue);
+
/// Helper union that holds any possible option type.
union ValueTy {
bool Bool;
@@ -317,12 +327,26 @@ struct BaseConfigurationOption {
}
///}
+ /// Set and get of the string-list value. Only valid if it is a string-list
+ /// option.
+ ///{
+ void setStringList(ArrayRef<StringRef> Values) {
+ assert(Kind == STRING_LIST && "Not a string list!");
+ StringList.assign(Values.begin(), Values.end());
+ }
+ ArrayRef<StringRef> getStringList() const {
+ assert(Kind == STRING_LIST && "Not a string list!");
+ return StringList;
+ }
+ ///}
+
/// The information of the option.
///{
StringRef Name;
StringRef Description;
KindTy Kind;
ValueTy Value = {0};
+ SmallVector<StringRef> StringList;
///}
/// Construct a base configuration option.
@@ -374,6 +398,9 @@ struct LLVM_ABI InstrumentationConfig {
*this, "runtime_bitcode", "Link runtime bitcode", "");
InlineRuntimeEagerly = BaseConfigurationOption::createBoolOption(
*this, "inline_runtime", "Inline runtime function calls eagerly", true);
+ RuntimeExportSymbols = BaseConfigurationOption::createStringListOption(
+ *this, "runtime_export_symbols",
+ "Runtime symbols that remain externally visible after linking.", {});
populate(IIRB);
}
@@ -447,6 +474,7 @@ struct LLVM_ABI InstrumentationConfig {
std::unique_ptr<BaseConfigurationOption> GPUEnabled;
std::unique_ptr<BaseConfigurationOption> RuntimeBitcode;
std::unique_ptr<BaseConfigurationOption> InlineRuntimeEagerly;
+ std::unique_ptr<BaseConfigurationOption> RuntimeExportSymbols;
/// The map registered instrumentation opportunities. The map is indexed by
/// the instrumentation location kind and then by the opportunity name. Notice
diff --git a/llvm/lib/Transforms/IPO/Instrumentor.cpp b/llvm/lib/Transforms/IPO/Instrumentor.cpp
index bfa22199ef9f6..4bd1af2312af8 100644
--- a/llvm/lib/Transforms/IPO/Instrumentor.cpp
+++ b/llvm/lib/Transforms/IPO/Instrumentor.cpp
@@ -262,8 +262,13 @@ void InstrumentorImpl::linkRuntime() {
}
auto InternalizeCallback = [&](Module &M, const StringSet<> &GVS) {
- internalizeModule(M, [&GVS](const GlobalValue &GV) {
- return !GV.hasName() || !GVS.count(GV.getName());
+ StringSet<> RuntimeExports;
+ for (StringRef Name : IConf.RuntimeExportSymbols->getStringList())
+ RuntimeExports.insert(Name);
+
+ internalizeModule(M, [&GVS, &RuntimeExports](const GlobalValue &GV) {
+ return !GV.hasName() || !GVS.count(GV.getName()) ||
+ RuntimeExports.count(GV.getName());
});
};
@@ -629,6 +634,17 @@ BaseConfigurationOption::createStringOption(InstrumentationConfig &IConf,
return BCO;
}
+std::unique_ptr<BaseConfigurationOption>
+BaseConfigurationOption::createStringListOption(
+ InstrumentationConfig &IConf, StringRef Name, StringRef Description,
+ ArrayRef<StringRef> DefaultValue) {
+ auto BCO = std::make_unique<BaseConfigurationOption>(Name, Description,
+ STRING_LIST);
+ BCO->setStringList(DefaultValue);
+ IConf.addBaseChoice(BCO.get());
+ return BCO;
+}
+
void InstrumentationConfig::populate(InstrumentorIRBuilderTy &IIRB) {
/// List of all instrumentation opportunities.
BasePointerIO::populate(*this, IIRB);
diff --git a/llvm/lib/Transforms/IPO/InstrumentorConfigFile.cpp b/llvm/lib/Transforms/IPO/InstrumentorConfigFile.cpp
index a2600b0551b65..083eca05b00f4 100644
--- a/llvm/lib/Transforms/IPO/InstrumentorConfigFile.cpp
+++ b/llvm/lib/Transforms/IPO/InstrumentorConfigFile.cpp
@@ -69,6 +69,12 @@ void writeConfigToJSON(InstrumentationConfig &IConf, StringRef OutputFile,
case BaseConfigurationOption::BOOLEAN:
J.attribute(BaseCO->Name, BaseCO->getBool());
break;
+ case BaseConfigurationOption::STRING_LIST:
+ J.attributeArray(BaseCO->Name, [&] {
+ for (StringRef Value : BaseCO->getStringList())
+ J.value(Value);
+ });
+ break;
}
if (!BaseCO->Description.empty())
J.attribute(std::string(BaseCO->Name) + ".description",
@@ -195,6 +201,35 @@ bool readConfigFromJSON(InstrumentationConfig &IConf, StringRef InputFile,
DS_Warning));
}
break;
+ case BaseConfigurationOption::STRING_LIST: {
+ auto *Values = ObjIt.second.getAsArray();
+ if (!Values) {
+ Ctx.diagnose(DiagnosticInfoInstrumentation(
+ Twine("configuration key '") + StringRef(ObjIt.first) +
+ Twine("' expects an array of strings, value ignored"),
+ DS_Warning));
+ break;
+ }
+
+ SmallVector<StringRef> Strings;
+ bool AllStrings = true;
+ for (const json::Value &Value : *Values) {
+ auto String = Value.getAsString();
+ if (!String) {
+ AllStrings = false;
+ break;
+ }
+ Strings.push_back(IConf.SS.save(*String));
+ }
+ if (AllStrings)
+ BO->setStringList(Strings);
+ else
+ Ctx.diagnose(DiagnosticInfoInstrumentation(
+ Twine("configuration key '") + StringRef(ObjIt.first) +
+ Twine("' expects an array of strings, value ignored"),
+ DS_Warning));
+ break;
+ }
}
} else if (!StringRef(ObjIt.first).ends_with(".description")) {
std::string Diag = "configuration key '" + ObjIt.first.str() +
diff --git a/llvm/test/Instrumentation/Instrumentor/default_config.json b/llvm/test/Instrumentation/Instrumentor/default_config.json
index fc712537e5e22..06057833ed8a6 100644
--- a/llvm/test/Instrumentation/Instrumentor/default_config.json
+++ b/llvm/test/Instrumentation/Instrumentor/default_config.json
@@ -17,7 +17,9 @@
"runtime_bitcode": "",
"runtime_bitcode.description": "Link runtime bitcode",
"inline_runtime": true,
- "inline_runtime.description": "Inline runtime function calls eagerly"
+ "inline_runtime.description": "Inline runtime function calls eagerly",
+ "runtime_export_symbols": [],
+ "runtime_export_symbols.description": "Runtime symbols that remain externally visible after linking."
},
"module_pre": {
"module": {
@@ -446,4 +448,4 @@
"id.description": "A unique ID associated with the given instrumentor call"
}
}
-}
\ No newline at end of file
+}
>From 314f775d7f726d01da9b25721739afe26c6d60c1 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Mon, 27 Jul 2026 16:48:28 -0700
Subject: [PATCH 2/8] [Instrumentor] Add runtime exports test
---
.../Instrumentor/runtime_export_symbols.ll | 14 ++++++++++++++
.../runtime_export_symbols_config.json | 15 +++++++++++++++
.../runtimes/runtime_export_symbols_rt.ll | 14 ++++++++++++++
3 files changed, 43 insertions(+)
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
new file mode 100644
index 0000000000000..5e822baa0a7a7
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
@@ -0,0 +1,14 @@
+; RUN: llvm-as %S/runtimes/runtime_export_symbols_rt.ll -o runtime_export_symbols_rt.bc
+; RUN: opt < %s -passes=instrumentor -instrumentor-read-config-files=%S/runtime_export_symbols_config.json -S | FileCheck %s
+
+; CHECK: @runtime_export = linkonce_odr global i32 0, comdat, align 4
+; CHECK: @runtime_internal = internal global i32 0, align 4
+
+ at runtime_export = external global i32
+ at runtime_internal = external global i32
+
+define i32 @test(i32 %lhs, i32 %rhs) {
+entry:
+ %result = add i32 %lhs, %rhs
+ ret i32 %result
+}
\ No newline at end of file
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json
new file mode 100644
index 0000000000000..2519dfb3796b3
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json
@@ -0,0 +1,15 @@
+{
+ "configuration": {
+ "runtime_prefix": "__runtime_export_",
+ "runtime_bitcode": "runtime_export_symbols_rt.bc",
+ "inline_runtime": false,
+ "runtime_export_symbols": ["runtime_export"]
+ },
+ "instruction_pre": {
+ "numeric": {
+ "enabled": true,
+ "size": true,
+ "id": true
+ }
+ }
+}
\ No newline at end of file
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
new file mode 100644
index 0000000000000..342dd718271b9
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
@@ -0,0 +1,14 @@
+$runtime_export = comdat any
+$runtime_internal = comdat any
+
+ at runtime_export = linkonce_odr global i32 0, comdat, align 4
+ at runtime_internal = linkonce_odr global i32 0, comdat, align 4
+
+define void @__runtime_export_pre_numeric(i32 %size, i32 %id) {
+entry:
+ %export = load i32, ptr @runtime_export, align 4
+ %internal = load i32, ptr @runtime_internal, align 4
+ %sum = add i32 %export, %internal
+ store i32 %sum, ptr @runtime_internal, align 4
+ ret void
+}
\ No newline at end of file
>From 7d319f01dbd7dab2c3ba7420e55dea99c008f534 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Fri, 14 Aug 2026 07:34:06 -0700
Subject: [PATCH 3/8] [Instrumentor] Format
---
llvm/include/llvm/Transforms/IPO/Instrumentor.h | 4 ++--
llvm/lib/Transforms/IPO/Instrumentor.cpp | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/Transforms/IPO/Instrumentor.h b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
index 6fbc620a7c4a9..915c3f3e35476 100644
--- a/llvm/include/llvm/Transforms/IPO/Instrumentor.h
+++ b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
@@ -399,8 +399,8 @@ struct LLVM_ABI InstrumentationConfig {
InlineRuntimeEagerly = BaseConfigurationOption::createBoolOption(
*this, "inline_runtime", "Inline runtime function calls eagerly", true);
RuntimeExportSymbols = BaseConfigurationOption::createStringListOption(
- *this, "runtime_export_symbols",
- "Runtime symbols that remain externally visible after linking.", {});
+ *this, "runtime_export_symbols",
+ "Runtime symbols that remain externally visible after linking.", {});
populate(IIRB);
}
diff --git a/llvm/lib/Transforms/IPO/Instrumentor.cpp b/llvm/lib/Transforms/IPO/Instrumentor.cpp
index 4bd1af2312af8..fc5b8a9845bef 100644
--- a/llvm/lib/Transforms/IPO/Instrumentor.cpp
+++ b/llvm/lib/Transforms/IPO/Instrumentor.cpp
@@ -638,8 +638,8 @@ std::unique_ptr<BaseConfigurationOption>
BaseConfigurationOption::createStringListOption(
InstrumentationConfig &IConf, StringRef Name, StringRef Description,
ArrayRef<StringRef> DefaultValue) {
- auto BCO = std::make_unique<BaseConfigurationOption>(Name, Description,
- STRING_LIST);
+ auto BCO =
+ std::make_unique<BaseConfigurationOption>(Name, Description, STRING_LIST);
BCO->setStringList(DefaultValue);
IConf.addBaseChoice(BCO.get());
return BCO;
>From 0dfbacd0bc6ce9c244e24d47497507eeb0122b72 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Fri, 14 Aug 2026 07:59:58 -0700
Subject: [PATCH 4/8] [Instrumentor] Add pre-runtime-link extension point
---
.../llvm/Transforms/IPO/Instrumentor.h | 7 ++
llvm/lib/Transforms/IPO/Instrumentor.cpp | 2 +
llvm/unittests/Transforms/IPO/CMakeLists.txt | 1 +
.../Transforms/IPO/InstrumentorTest.cpp | 94 +++++++++++++++++++
4 files changed, 104 insertions(+)
create mode 100644 llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
diff --git a/llvm/include/llvm/Transforms/IPO/Instrumentor.h b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
index 915c3f3e35476..34a4cea86d220 100644
--- a/llvm/include/llvm/Transforms/IPO/Instrumentor.h
+++ b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
@@ -407,6 +407,13 @@ struct LLVM_ABI InstrumentationConfig {
/// Populate the instrumentation opportunities.
virtual void populate(InstrumentorIRBuilderTy &IIRB);
+ /// Allow embedded users to extend the module after instrumentation and before
+ /// runtime bitcode linking.
+ virtual bool instrumentBeforeRuntimeLink(Module &,
+ InstrumentorIRBuilderTy &) {
+ return false;
+ }
+
/// Get the runtime prefix for the instrumentation runtime functions.
StringRef getRTName() const { return RuntimePrefix->getString(); }
diff --git a/llvm/lib/Transforms/IPO/Instrumentor.cpp b/llvm/lib/Transforms/IPO/Instrumentor.cpp
index fc5b8a9845bef..44e95c32e1d43 100644
--- a/llvm/lib/Transforms/IPO/Instrumentor.cpp
+++ b/llvm/lib/Transforms/IPO/Instrumentor.cpp
@@ -541,6 +541,8 @@ bool InstrumentorImpl::instrument() {
for (Function &Fn : M)
Changed |= instrumentFunction(Fn);
+ Changed |= IConf.instrumentBeforeRuntimeLink(M, IIRB);
+
linkRuntime();
return Changed;
diff --git a/llvm/unittests/Transforms/IPO/CMakeLists.txt b/llvm/unittests/Transforms/IPO/CMakeLists.txt
index 5b45191afc711..a54bf05b047b5 100644
--- a/llvm/unittests/Transforms/IPO/CMakeLists.txt
+++ b/llvm/unittests/Transforms/IPO/CMakeLists.txt
@@ -14,5 +14,6 @@ add_llvm_unittest(IPOTests
AttributorTest.cpp
FunctionSpecializationTest.cpp
ImportIDTableTests.cpp
+ InstrumentorTest.cpp
MergeFunctionsTest.cpp
)
diff --git a/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp b/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
new file mode 100644
index 0000000000000..4f89ada4dbb35
--- /dev/null
+++ b/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
@@ -0,0 +1,94 @@
+//===- InstrumentorTest.cpp - Unit tests for InstrumentorPass -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO/Instrumentor.h"
+
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/raw_ostream.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::instrumentor;
+
+namespace {
+
+class PreRuntimeLinkConfig final : public InstrumentationConfig {
+public:
+ explicit PreRuntimeLinkConfig(StringRef RuntimePath)
+ : RuntimePath(RuntimePath) {}
+
+ void populate(InstrumentorIRBuilderTy &) override {
+ RuntimeBitcode->setString(RuntimePath);
+ }
+
+ bool instrumentBeforeRuntimeLink(Module &M,
+ InstrumentorIRBuilderTy &) override {
+ LLVMContext &Ctx = M.getContext();
+ Type *Ty = Type::getInt1Ty(Ctx);
+ Constant *SawRuntime =
+ ConstantInt::get(Ty, M.getNamedGlobal("runtime_marker") != nullptr);
+ new GlobalVariable(M, Ty, false, GlobalValue::ExternalLinkage, SawRuntime,
+ "hook_saw_runtime");
+ return true;
+ }
+
+private:
+ std::string RuntimePath;
+};
+
+std::unique_ptr<Module> parseModule(StringRef IR, LLVMContext &Ctx) {
+ SMDiagnostic Err;
+ std::unique_ptr<Module> M = parseAssemblyString(IR, Err, Ctx);
+ EXPECT_TRUE(M);
+ return M;
+}
+
+TEST(InstrumentorTest, RunsHookBeforeRuntimeLink) {
+ SmallString<128> RuntimePath;
+ int FD;
+ ASSERT_FALSE(sys::fs::createTemporaryFile("instrumentor-runtime", "ll", FD,
+ RuntimePath));
+ auto RemoveRuntime = make_scope_exit([&] { sys::fs::remove(RuntimePath); });
+
+ raw_fd_ostream OS(FD, true);
+ OS << "@runtime_marker = global i32 0\n";
+ OS.close();
+
+ LLVMContext Ctx;
+ std::unique_ptr<Module> M = parseModule(R"ir(
+ define void @test() {
+ entry:
+ ret void
+ }
+ )ir",
+ Ctx);
+ ASSERT_TRUE(M);
+
+ PreRuntimeLinkConfig Config(RuntimePath);
+ ModuleAnalysisManager MAM;
+ InstrumentorPass Pass(nullptr, &Config);
+ Pass.run(*M, MAM);
+
+ GlobalVariable *HookSawRuntime = M->getNamedGlobal("hook_saw_runtime");
+ ASSERT_NE(HookSawRuntime, nullptr);
+ auto *Initializer = dyn_cast<ConstantInt>(HookSawRuntime->getInitializer());
+ ASSERT_NE(Initializer, nullptr);
+ EXPECT_TRUE(Initializer->isZero());
+ EXPECT_NE(M->getNamedGlobal("runtime_marker"), nullptr);
+}
+
+} // namespace
>From 82bd21f5137f40a59cf43752d69bc50ad17e73e5 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Fri, 14 Aug 2026 08:54:19 -0700
Subject: [PATCH 5/8] [Instrumentor] Fix test
---
llvm/unittests/Transforms/IPO/InstrumentorTest.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp b/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
index 4f89ada4dbb35..d3b9df14c0994 100644
--- a/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
+++ b/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
@@ -18,6 +18,7 @@
#include "llvm/IR/PassManager.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Support/raw_ostream.h"
#include "gtest/gtest.h"
@@ -62,7 +63,7 @@ TEST(InstrumentorTest, RunsHookBeforeRuntimeLink) {
int FD;
ASSERT_FALSE(sys::fs::createTemporaryFile("instrumentor-runtime", "ll", FD,
RuntimePath));
- auto RemoveRuntime = make_scope_exit([&] { sys::fs::remove(RuntimePath); });
+ scope_exit RemoveRuntime([&] { sys::fs::remove(RuntimePath); });
raw_fd_ostream OS(FD, true);
OS << "@runtime_marker = global i32 0\n";
@@ -80,7 +81,7 @@ TEST(InstrumentorTest, RunsHookBeforeRuntimeLink) {
PreRuntimeLinkConfig Config(RuntimePath);
ModuleAnalysisManager MAM;
- InstrumentorPass Pass(nullptr, &Config);
+ InstrumentorPass Pass(/*FS=*/nullptr, &Config, /*IIRB=*/nullptr);
Pass.run(*M, MAM);
GlobalVariable *HookSawRuntime = M->getNamedGlobal("hook_saw_runtime");
>From f5ede09f8127021a4d4977c8b8b7dfb5fa364f51 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Fri, 14 Aug 2026 10:56:08 -0700
Subject: [PATCH 6/8] [Instrumentor] Use weak linkage for runtime exports
---
llvm/lib/Transforms/IPO/Instrumentor.cpp | 7 +++++++
.../Instrumentor/runtime_export_symbols.ll | 4 ++--
.../Instrumentor/runtimes/runtime_export_symbols_rt.ll | 9 +++------
3 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/IPO/Instrumentor.cpp b/llvm/lib/Transforms/IPO/Instrumentor.cpp
index 44e95c32e1d43..13b911ea7ad9f 100644
--- a/llvm/lib/Transforms/IPO/Instrumentor.cpp
+++ b/llvm/lib/Transforms/IPO/Instrumentor.cpp
@@ -270,6 +270,13 @@ void InstrumentorImpl::linkRuntime() {
return !GV.hasName() || !GVS.count(GV.getName()) ||
RuntimeExports.count(GV.getName());
});
+
+ for (StringRef Name : IConf.RuntimeExportSymbols->getStringList()) {
+ GlobalValue *GV = M.getNamedValue(Name);
+ if (!GV || GV->isDeclarationForLinker() || GV->hasLocalLinkage())
+ continue;
+ GV->setLinkage(GlobalValue::WeakAnyLinkage);
+ }
};
if (Linker::linkModules(M, std::move(RTM), 0, InternalizeCallback)) {
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
index 5e822baa0a7a7..ad804f97ec53b 100644
--- a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
+++ b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
@@ -1,7 +1,7 @@
; RUN: llvm-as %S/runtimes/runtime_export_symbols_rt.ll -o runtime_export_symbols_rt.bc
; RUN: opt < %s -passes=instrumentor -instrumentor-read-config-files=%S/runtime_export_symbols_config.json -S | FileCheck %s
-; CHECK: @runtime_export = linkonce_odr global i32 0, comdat, align 4
+; CHECK: @runtime_export = weak global i32 0, align 4
; CHECK: @runtime_internal = internal global i32 0, align 4
@runtime_export = external global i32
@@ -11,4 +11,4 @@ define i32 @test(i32 %lhs, i32 %rhs) {
entry:
%result = add i32 %lhs, %rhs
ret i32 %result
-}
\ No newline at end of file
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
index 342dd718271b9..c13a521f24ff9 100644
--- a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
+++ b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
@@ -1,8 +1,5 @@
-$runtime_export = comdat any
-$runtime_internal = comdat any
-
- at runtime_export = linkonce_odr global i32 0, comdat, align 4
- at runtime_internal = linkonce_odr global i32 0, comdat, align 4
+ at runtime_export = global i32 0, align 4
+ at runtime_internal = global i32 0, align 4
define void @__runtime_export_pre_numeric(i32 %size, i32 %id) {
entry:
@@ -11,4 +8,4 @@ entry:
%sum = add i32 %export, %internal
store i32 %sum, ptr @runtime_internal, align 4
ret void
-}
\ No newline at end of file
+}
>From 6e4f29a196aa7591c27c9cc9173476500c9d8e17 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Wed, 19 Aug 2026 07:49:15 -0700
Subject: [PATCH 7/8] [Instrumentor] Support multiple runtime bitcodes
---
.../llvm/Transforms/IPO/Instrumentor.h | 10 ++--
llvm/lib/Transforms/IPO/Instrumentor.cpp | 52 +++++++------------
.../Instrumentor/default_config.json | 8 ++-
.../Instrumentor/inline_runtime_config.json | 2 +-
.../Instrumentor/runtime_bitcodes.ll | 17 ++++++
.../Instrumentor/runtime_bitcodes_config.json | 17 ++++++
.../Instrumentor/runtime_export_symbols.ll | 14 -----
.../runtime_export_symbols_config.json | 15 ------
.../runtimes/runtime_bitcodes_callbacks_rt.ll | 11 ++++
.../runtimes/runtime_bitcodes_state_rt.ll | 1 +
.../runtimes/runtime_export_symbols_rt.ll | 11 ----
.../Transforms/IPO/InstrumentorTest.cpp | 2 +-
12 files changed, 74 insertions(+), 86 deletions(-)
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtime_bitcodes.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtime_bitcodes_config.json
delete mode 100644 llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
delete mode 100644 llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_callbacks_rt.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_state_rt.ll
delete mode 100644 llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
diff --git a/llvm/include/llvm/Transforms/IPO/Instrumentor.h b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
index 34a4cea86d220..202b132ec3962 100644
--- a/llvm/include/llvm/Transforms/IPO/Instrumentor.h
+++ b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
@@ -394,13 +394,10 @@ struct LLVM_ABI InstrumentationConfig {
*this, "host_enabled", "Instrument non-GPU targets", true);
GPUEnabled = BaseConfigurationOption::createBoolOption(
*this, "gpu_enabled", "Instrument GPU targets", true);
- RuntimeBitcode = BaseConfigurationOption::createStringOption(
- *this, "runtime_bitcode", "Link runtime bitcode", "");
+ RuntimeBitcodes = BaseConfigurationOption::createStringListOption(
+ *this, "runtime_bitcodes", "Link runtime bitcode files", {});
InlineRuntimeEagerly = BaseConfigurationOption::createBoolOption(
*this, "inline_runtime", "Inline runtime function calls eagerly", true);
- RuntimeExportSymbols = BaseConfigurationOption::createStringListOption(
- *this, "runtime_export_symbols",
- "Runtime symbols that remain externally visible after linking.", {});
populate(IIRB);
}
@@ -479,9 +476,8 @@ struct LLVM_ABI InstrumentationConfig {
std::unique_ptr<BaseConfigurationOption> FunctionRegex;
std::unique_ptr<BaseConfigurationOption> HostEnabled;
std::unique_ptr<BaseConfigurationOption> GPUEnabled;
- std::unique_ptr<BaseConfigurationOption> RuntimeBitcode;
+ std::unique_ptr<BaseConfigurationOption> RuntimeBitcodes;
std::unique_ptr<BaseConfigurationOption> InlineRuntimeEagerly;
- std::unique_ptr<BaseConfigurationOption> RuntimeExportSymbols;
/// The map registered instrumentation opportunities. The map is indexed by
/// the instrumentation location kind and then by the opportunity name. Notice
diff --git a/llvm/lib/Transforms/IPO/Instrumentor.cpp b/llvm/lib/Transforms/IPO/Instrumentor.cpp
index 13b911ea7ad9f..f16ee0c23d082 100644
--- a/llvm/lib/Transforms/IPO/Instrumentor.cpp
+++ b/llvm/lib/Transforms/IPO/Instrumentor.cpp
@@ -50,7 +50,6 @@
#include "llvm/Support/Regex.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Transforms/IPO/InstrumentorUtils.h"
-#include "llvm/Transforms/IPO/Internalize.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include "llvm/Transforms/Utils/PromoteMemToReg.h"
@@ -247,42 +246,31 @@ static Regex createRegex(StringRef Str, StringRef Name, LLVMContext &Ctx) {
}
void InstrumentorImpl::linkRuntime() {
- const auto RuntimeBitcode = IConf.RuntimeBitcode->getString();
- if (RuntimeBitcode.empty())
+ ArrayRef<StringRef> RuntimeBitcodes = IConf.RuntimeBitcodes->getStringList();
+ if (RuntimeBitcodes.empty())
return;
- SMDiagnostic Err;
- auto RTM = parseIRFile(RuntimeBitcode, Err, M.getContext());
- if (!RTM) {
- IIRB.Ctx.diagnose(DiagnosticInfoInstrumentation(
- Twine("Failed to parse runtime bitcode file '") + RuntimeBitcode +
- Twine("':\n") + M.getName(),
- DS_Error));
- return;
- }
-
- auto InternalizeCallback = [&](Module &M, const StringSet<> &GVS) {
- StringSet<> RuntimeExports;
- for (StringRef Name : IConf.RuntimeExportSymbols->getStringList())
- RuntimeExports.insert(Name);
-
- internalizeModule(M, [&GVS, &RuntimeExports](const GlobalValue &GV) {
- return !GV.hasName() || !GVS.count(GV.getName()) ||
- RuntimeExports.count(GV.getName());
- });
+ for (StringRef RuntimeBitcode : RuntimeBitcodes) {
+ if (RuntimeBitcode.empty())
+ continue;
- for (StringRef Name : IConf.RuntimeExportSymbols->getStringList()) {
- GlobalValue *GV = M.getNamedValue(Name);
- if (!GV || GV->isDeclarationForLinker() || GV->hasLocalLinkage())
- continue;
- GV->setLinkage(GlobalValue::WeakAnyLinkage);
+ SMDiagnostic Err;
+ auto RTM = parseIRFile(RuntimeBitcode, Err, M.getContext());
+ if (!RTM) {
+ IIRB.Ctx.diagnose(DiagnosticInfoInstrumentation(
+ Twine("Failed to parse runtime bitcode file '") + RuntimeBitcode +
+ Twine("':\n") + M.getName(),
+ DS_Error));
+ return;
}
- };
- if (Linker::linkModules(M, std::move(RTM), 0, InternalizeCallback)) {
- IIRB.Ctx.diagnose(DiagnosticInfoInstrumentation(
- "Failed to link in runtime bitcode", DS_Error));
- return;
+ if (Linker::linkModules(M, std::move(RTM))) {
+ IIRB.Ctx.diagnose(DiagnosticInfoInstrumentation(
+ Twine("Failed to link in runtime bitcode file '") + RuntimeBitcode +
+ "'",
+ DS_Error));
+ return;
+ }
}
if (!IConf.InlineRuntimeEagerly->getBool())
diff --git a/llvm/test/Instrumentation/Instrumentor/default_config.json b/llvm/test/Instrumentation/Instrumentor/default_config.json
index 06057833ed8a6..362e866f7882e 100644
--- a/llvm/test/Instrumentation/Instrumentor/default_config.json
+++ b/llvm/test/Instrumentation/Instrumentor/default_config.json
@@ -14,12 +14,10 @@
"host_enabled.description": "Instrument non-GPU targets",
"gpu_enabled": true,
"gpu_enabled.description": "Instrument GPU targets",
- "runtime_bitcode": "",
- "runtime_bitcode.description": "Link runtime bitcode",
+ "runtime_bitcodes": [],
+ "runtime_bitcodes.description": "Link runtime bitcode files",
"inline_runtime": true,
- "inline_runtime.description": "Inline runtime function calls eagerly",
- "runtime_export_symbols": [],
- "runtime_export_symbols.description": "Runtime symbols that remain externally visible after linking."
+ "inline_runtime.description": "Inline runtime function calls eagerly"
},
"module_pre": {
"module": {
diff --git a/llvm/test/Instrumentation/Instrumentor/inline_runtime_config.json b/llvm/test/Instrumentation/Instrumentor/inline_runtime_config.json
index 3f459240deacd..9f406c3be2878 100644
--- a/llvm/test/Instrumentation/Instrumentor/inline_runtime_config.json
+++ b/llvm/test/Instrumentation/Instrumentor/inline_runtime_config.json
@@ -1,7 +1,7 @@
{
"configuration": {
"runtime_prefix": "__bytes_computed_",
- "runtime_bitcode": "bytes_computed_rt.bc",
+ "runtime_bitcodes": ["bytes_computed_rt.bc"],
"inline_runtime": true
},
"instruction_pre": {
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_bitcodes.ll b/llvm/test/Instrumentation/Instrumentor/runtime_bitcodes.ll
new file mode 100644
index 0000000000000..14b2cfcdcbcf2
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtime_bitcodes.ll
@@ -0,0 +1,17 @@
+; This test checks that Instrumentor links multiple runtime bitcode modules.
+; The callback runtime uses state from the preceding runtime module.
+; RUN: llvm-as %S/runtimes/runtime_bitcodes_state_rt.ll -o runtime_bitcodes_state_rt.bc
+; RUN: llvm-as %S/runtimes/runtime_bitcodes_callbacks_rt.ll -o runtime_bitcodes_callbacks_rt.bc
+; RUN: opt < %s -passes=instrumentor -instrumentor-read-config-files=%S/runtime_bitcodes_config.json -S | FileCheck %s
+
+; CHECK-DAG: @runtime_state = protected global i32 0, align 4
+; CHECK-DAG: @runtime_private_state = internal global i32 0, align 4
+; CHECK-DAG: define protected void @__runtime_bitcodes_pre_numeric(
+
+ at runtime_state = external global i32
+
+define i32 @test(i32 %lhs, i32 %rhs) {
+entry:
+ %result = add i32 %lhs, %rhs
+ ret i32 %result
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_bitcodes_config.json b/llvm/test/Instrumentation/Instrumentor/runtime_bitcodes_config.json
new file mode 100644
index 0000000000000..822f4f6b04ea2
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtime_bitcodes_config.json
@@ -0,0 +1,17 @@
+{
+ "configuration": {
+ "runtime_prefix": "__runtime_bitcodes_",
+ "runtime_bitcodes": [
+ "runtime_bitcodes_state_rt.bc",
+ "runtime_bitcodes_callbacks_rt.bc"
+ ],
+ "inline_runtime": false
+ },
+ "instruction_pre": {
+ "numeric": {
+ "enabled": true,
+ "size": true,
+ "id": true
+ }
+ }
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
deleted file mode 100644
index ad804f97ec53b..0000000000000
--- a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
+++ /dev/null
@@ -1,14 +0,0 @@
-; RUN: llvm-as %S/runtimes/runtime_export_symbols_rt.ll -o runtime_export_symbols_rt.bc
-; RUN: opt < %s -passes=instrumentor -instrumentor-read-config-files=%S/runtime_export_symbols_config.json -S | FileCheck %s
-
-; CHECK: @runtime_export = weak global i32 0, align 4
-; CHECK: @runtime_internal = internal global i32 0, align 4
-
- at runtime_export = external global i32
- at runtime_internal = external global i32
-
-define i32 @test(i32 %lhs, i32 %rhs) {
-entry:
- %result = add i32 %lhs, %rhs
- ret i32 %result
-}
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json
deleted file mode 100644
index 2519dfb3796b3..0000000000000
--- a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "configuration": {
- "runtime_prefix": "__runtime_export_",
- "runtime_bitcode": "runtime_export_symbols_rt.bc",
- "inline_runtime": false,
- "runtime_export_symbols": ["runtime_export"]
- },
- "instruction_pre": {
- "numeric": {
- "enabled": true,
- "size": true,
- "id": true
- }
- }
-}
\ No newline at end of file
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_callbacks_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_callbacks_rt.ll
new file mode 100644
index 0000000000000..697f7be6b79bb
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_callbacks_rt.ll
@@ -0,0 +1,11 @@
+ at runtime_state = external global i32
+ at runtime_private_state = internal global i32 0, align 4
+
+define protected void @__runtime_bitcodes_pre_numeric(i32 %size, i32 %id) {
+entry:
+ %state = load i32, ptr @runtime_state, align 4
+ %private_state = load i32, ptr @runtime_private_state, align 4
+ %sum = add i32 %state, %private_state
+ store i32 %sum, ptr @runtime_private_state, align 4
+ ret void
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_state_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_state_rt.ll
new file mode 100644
index 0000000000000..c11f53367a47a
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_state_rt.ll
@@ -0,0 +1 @@
+ at runtime_state = protected global i32 0, align 4
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
deleted file mode 100644
index c13a521f24ff9..0000000000000
--- a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
+++ /dev/null
@@ -1,11 +0,0 @@
- at runtime_export = global i32 0, align 4
- at runtime_internal = global i32 0, align 4
-
-define void @__runtime_export_pre_numeric(i32 %size, i32 %id) {
-entry:
- %export = load i32, ptr @runtime_export, align 4
- %internal = load i32, ptr @runtime_internal, align 4
- %sum = add i32 %export, %internal
- store i32 %sum, ptr @runtime_internal, align 4
- ret void
-}
diff --git a/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp b/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
index d3b9df14c0994..504f68214bda1 100644
--- a/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
+++ b/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
@@ -33,7 +33,7 @@ class PreRuntimeLinkConfig final : public InstrumentationConfig {
: RuntimePath(RuntimePath) {}
void populate(InstrumentorIRBuilderTy &) override {
- RuntimeBitcode->setString(RuntimePath);
+ RuntimeBitcodes->setStringList({StringRef(RuntimePath)});
}
bool instrumentBeforeRuntimeLink(Module &M,
>From cb56fc4e12419662fcd2e4cc0892f208b8ab1d00 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Wed, 19 Aug 2026 15:53:24 -0700
Subject: [PATCH 8/8] [Offload][InputGenGPU] Add GPU instrumentation pass
policy
---
.../include/llvm/Transforms/IPO/InputGenGPU.h | 24 +++++++
llvm/lib/Passes/PassBuilder.cpp | 1 +
llvm/lib/Passes/PassRegistry.def | 1 +
llvm/lib/Transforms/IPO/CMakeLists.txt | 1 +
llvm/lib/Transforms/IPO/InputGenGPU.cpp | 72 +++++++++++++++++++
.../Instrumentor/inputgen_gpu_host.ll | 18 +++++
.../Instrumentor/inputgen_gpu_post_load.ll | 20 ++++++
.../Instrumentor/inputgen_gpu_runtime_link.ll | 17 +++++
.../runtimes/inputgen_gpu_callbacks_rt.ll | 9 +++
9 files changed, 163 insertions(+)
create mode 100644 llvm/include/llvm/Transforms/IPO/InputGenGPU.h
create mode 100644 llvm/lib/Transforms/IPO/InputGenGPU.cpp
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_host.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_post_load.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_runtime_link.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_callbacks_rt.ll
diff --git a/llvm/include/llvm/Transforms/IPO/InputGenGPU.h b/llvm/include/llvm/Transforms/IPO/InputGenGPU.h
new file mode 100644
index 0000000000000..0e9af03fb7f81
--- /dev/null
+++ b/llvm/include/llvm/Transforms/IPO/InputGenGPU.h
@@ -0,0 +1,24 @@
+//===-- InputGenGPU.h - InputGen GPU instrumentation pass -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_IPO_INPUTGENGPU_H
+#define LLVM_TRANSFORMS_IPO_INPUTGENGPU_H
+
+#include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
+
+namespace llvm {
+
+class InputGenGPUPass : public RequiredPassInfoMixin<InputGenGPUPass> {
+public:
+ LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_IPO_INPUTGENGPU_H
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 6abaa77f9065f..54e4394713b59 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -241,6 +241,7 @@
#include "llvm/Transforms/IPO/GlobalSplit.h"
#include "llvm/Transforms/IPO/HotColdSplitting.h"
#include "llvm/Transforms/IPO/InferFunctionAttrs.h"
+#include "llvm/Transforms/IPO/InputGenGPU.h"
#include "llvm/Transforms/IPO/Instrumentor.h"
#include "llvm/Transforms/IPO/Internalize.h"
#include "llvm/Transforms/IPO/LowerTypeTests.h"
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 177d8ecd3508d..c838cca01ecee 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -95,6 +95,7 @@ MODULE_PASS("hipstdpar-select-accelerator-code",
HipStdParAcceleratorCodeSelectionPass())
MODULE_PASS("hotcoldsplit", HotColdSplittingPass())
MODULE_PASS("inferattrs", InferFunctionAttrsPass())
+MODULE_PASS("inputgen-gpu", InputGenGPUPass())
MODULE_PASS("inliner-ml-advisor-release",
ModuleInlinerWrapperPass(getInlineParams(), true, {},
InliningAdvisorMode::Release, 0))
diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt
index ca0e140264829..4871873cc1f3b 100644
--- a/llvm/lib/Transforms/IPO/CMakeLists.txt
+++ b/llvm/lib/Transforms/IPO/CMakeLists.txt
@@ -24,6 +24,7 @@ add_llvm_component_library(LLVMipo
GlobalSplit.cpp
HotColdSplitting.cpp
IPO.cpp
+ InputGenGPU.cpp
InferFunctionAttrs.cpp
Inliner.cpp
Instrumentor.cpp
diff --git a/llvm/lib/Transforms/IPO/InputGenGPU.cpp b/llvm/lib/Transforms/IPO/InputGenGPU.cpp
new file mode 100644
index 0000000000000..59ab811db85f8
--- /dev/null
+++ b/llvm/lib/Transforms/IPO/InputGenGPU.cpp
@@ -0,0 +1,72 @@
+//===-- InputGenGPU.cpp - InputGen GPU instrumentation pass ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO/InputGenGPU.h"
+#include "llvm/Transforms/IPO/Instrumentor.h"
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+using namespace llvm;
+using namespace llvm::instrumentor;
+
+#define DEBUG_TYPE "inputgen-gpu"
+
+static cl::list<std::string> InputGenGPURuntimeBitcodes(
+ "inputgen-gpu-runtime-bitcode",
+ cl::desc("InputGen GPU runtime bitcode file; may be repeated"),
+ cl::ZeroOrMore);
+
+namespace {
+
+class InputGenGPUConfig final : public InstrumentationConfig {
+ void populate(InstrumentorIRBuilderTy &IIRB) override {
+ InstrumentationConfig::populate(IIRB);
+
+ RuntimePrefix->setString("__ig_");
+ HostEnabled->setBool(false);
+ GPUEnabled->setBool(true);
+ SmallVector<StringRef> RuntimeBitcodeRefs;
+ for (StringRef RuntimeBitcode : InputGenGPURuntimeBitcodes)
+ RuntimeBitcodeRefs.push_back(RuntimeBitcode);
+ RuntimeBitcodes->setStringList(RuntimeBitcodeRefs);
+ InlineRuntimeEagerly->setBool(false);
+
+ for (auto &ChoiceMap : IChoices) {
+ for (auto &ChoiceIt : ChoiceMap) {
+ auto *IO = ChoiceIt.second;
+ IO->Enabled = false;
+ IO->Filter = "";
+ for (IRTArg &Arg : IO->IRTArgs)
+ Arg.Enabled = false;
+ }
+ }
+
+ auto *PostLoad =
+ IChoices[InstrumentationLocation::INSTRUCTION_POST].lookup("load");
+ if (!PostLoad)
+ return;
+
+ PostLoad->Enabled = true;
+ for (IRTArg &Arg : PostLoad->IRTArgs) {
+ Arg.Enabled = Arg.Name == "value" || Arg.Name == "value_size" ||
+ Arg.Name == "value_type_id" || Arg.Name == "id";
+ }
+ }
+};
+
+} // end anonymous namespace
+
+PreservedAnalyses InputGenGPUPass::run(Module &M, ModuleAnalysisManager &MAM) {
+ InputGenGPUConfig IConf;
+ InstrumentorIRBuilderTy IIRB(M);
+ return InstrumentorPass(/*FS=*/nullptr, &IConf, &IIRB).run(M, MAM);
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_host.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_host.ll
new file mode 100644
index 0000000000000..140e68aa98af4
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_host.ll
@@ -0,0 +1,18 @@
+; Verify the GPU policy leaves a host-targeted function unchanged.
+; RUN: opt < %s -passes=inputgen-gpu -S | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; The inputgen-gpu pass is GPU-only, so host IR should remain unchanged.
+; CHECK-LABEL: define i32 @vvv_foo(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %v = load i32, ptr %a, align 4
+; CHECK-NEXT: ret i32 %v
+; CHECK-NEXT: }
+
+define i32 @vvv_foo(ptr noundef %a) {
+entry:
+ %v = load i32, ptr %a, align 4
+ ret i32 %v
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_post_load.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_post_load.ll
new file mode 100644
index 0000000000000..066840ffb2221
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_post_load.ll
@@ -0,0 +1,20 @@
+; Verify the GPU policy replaces an integer load with the post-load callback.
+; RUN: opt < %s -passes=inputgen-gpu -S | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+; CHECK-NOT: __ig_pre_load
+; CHECK-NOT: __ig_post_store
+; CHECK-LABEL: define hidden i32 @vvv_foo(
+; CHECK: [[LOAD:%.*]] = load i32, ptr {{%.*}}, align 4
+; CHECK: [[EXT:%.*]] = zext i32 [[LOAD]] to i64
+; CHECK: [[CALL:%.*]] = call i64 @__ig_post_load(i64 [[EXT]], i64 4, i32 12, i32 -1)
+; CHECK: [[TRUNC:%.*]] = trunc i64 [[CALL]] to i32
+; CHECK: ret i32 [[TRUNC]]
+
+define hidden i32 @vvv_foo(ptr noundef %a) {
+entry:
+ %v = load i32, ptr %a, align 4
+ ret i32 %v
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_runtime_link.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_runtime_link.ll
new file mode 100644
index 0000000000000..ebca7c743f391
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_runtime_link.ll
@@ -0,0 +1,17 @@
+; Verify configured runtime bitcode provides the post-load callback definition.
+; RUN: llvm-as %S/runtimes/inputgen_gpu_callbacks_rt.ll -o %t.rt.bc
+; RUN: opt < %s -passes=inputgen-gpu -inputgen-gpu-runtime-bitcode=%t.rt.bc -S | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+; CHECK-NOT: @inputgen_
+; CHECK-LABEL: define hidden i32 @vvv_foo(
+; CHECK: call i64 @__ig_post_load(
+; CHECK-LABEL: define protected i64 @__ig_post_load(
+
+define hidden i32 @vvv_foo(ptr noundef %a) {
+entry:
+ %v = load i32, ptr %a, align 4
+ ret i32 %v
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_callbacks_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_callbacks_rt.ll
new file mode 100644
index 0000000000000..b9e73b5321da4
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_callbacks_rt.ll
@@ -0,0 +1,9 @@
+; Provide the callback-only runtime fixture used by the runtime-link test.
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+define protected i64 @__ig_post_load(i64 %value, i64 %value_size,
+ i32 %value_type_id, i32 %id) {
+entry:
+ ret i64 %value
+}
More information about the llvm-commits
mailing list