[llvm] [Offload][InputGenGPU] Generate GPU direct entry generation (PR #217497)
Vincent Arcila via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 22:12:18 PDT 2026
https://github.com/jandrovins updated https://github.com/llvm/llvm-project/pull/217497
>From 0fd94ee68387a62370a69e56c8bee44165c18662 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Mon, 27 Jul 2026 16:39:39 -0700
Subject: [PATCH 1/9] [Instrumentor] Add .json option for keeping symbols
visible after linking
---
.../llvm/Transforms/IPO/Instrumentor.h | 28 +++++++++++++++
llvm/lib/Transforms/IPO/Instrumentor.cpp | 20 +++++++++--
.../Transforms/IPO/InstrumentorConfigFile.cpp | 35 +++++++++++++++++++
.../Instrumentor/default_config.json | 6 ++--
4 files changed, 85 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/Transforms/IPO/Instrumentor.h b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
index 5da53ac0ada15..6fbc620a7c4a9 100644
--- a/llvm/include/llvm/Transforms/IPO/Instrumentor.h
+++ b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
@@ -13,10 +13,12 @@
#ifndef LLVM_TRANSFORMS_IPO_INSTRUMENTOR_H
#define LLVM_TRANSFORMS_IPO_INSTRUMENTOR_H
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/EnumeratedArray.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
@@ -273,6 +275,7 @@ struct BaseConfigurationOption {
enum KindTy {
STRING,
BOOLEAN,
+ STRING_LIST,
};
/// Create a boolean option with \p Name name, \p Description description and
@@ -287,6 +290,13 @@ struct BaseConfigurationOption {
createStringOption(InstrumentationConfig &IC, StringRef Name,
StringRef Description, StringRef DefaultValue);
+ /// Create a string-list option with \p Name name, \p Description
+ /// description and \p DefaultValue as string-list default value.
+ LLVM_ABI static std::unique_ptr<BaseConfigurationOption>
+ createStringListOption(InstrumentationConfig &IC, StringRef Name,
+ StringRef Description,
+ ArrayRef<StringRef> DefaultValue);
+
/// Helper union that holds any possible option type.
union ValueTy {
bool Bool;
@@ -317,12 +327,26 @@ struct BaseConfigurationOption {
}
///}
+ /// Set and get of the string-list value. Only valid if it is a string-list
+ /// option.
+ ///{
+ void setStringList(ArrayRef<StringRef> Values) {
+ assert(Kind == STRING_LIST && "Not a string list!");
+ StringList.assign(Values.begin(), Values.end());
+ }
+ ArrayRef<StringRef> getStringList() const {
+ assert(Kind == STRING_LIST && "Not a string list!");
+ return StringList;
+ }
+ ///}
+
/// The information of the option.
///{
StringRef Name;
StringRef Description;
KindTy Kind;
ValueTy Value = {0};
+ SmallVector<StringRef> StringList;
///}
/// Construct a base configuration option.
@@ -374,6 +398,9 @@ struct LLVM_ABI InstrumentationConfig {
*this, "runtime_bitcode", "Link runtime bitcode", "");
InlineRuntimeEagerly = BaseConfigurationOption::createBoolOption(
*this, "inline_runtime", "Inline runtime function calls eagerly", true);
+ RuntimeExportSymbols = BaseConfigurationOption::createStringListOption(
+ *this, "runtime_export_symbols",
+ "Runtime symbols that remain externally visible after linking.", {});
populate(IIRB);
}
@@ -447,6 +474,7 @@ struct LLVM_ABI InstrumentationConfig {
std::unique_ptr<BaseConfigurationOption> GPUEnabled;
std::unique_ptr<BaseConfigurationOption> RuntimeBitcode;
std::unique_ptr<BaseConfigurationOption> InlineRuntimeEagerly;
+ std::unique_ptr<BaseConfigurationOption> RuntimeExportSymbols;
/// The map registered instrumentation opportunities. The map is indexed by
/// the instrumentation location kind and then by the opportunity name. Notice
diff --git a/llvm/lib/Transforms/IPO/Instrumentor.cpp b/llvm/lib/Transforms/IPO/Instrumentor.cpp
index bfa22199ef9f6..4bd1af2312af8 100644
--- a/llvm/lib/Transforms/IPO/Instrumentor.cpp
+++ b/llvm/lib/Transforms/IPO/Instrumentor.cpp
@@ -262,8 +262,13 @@ void InstrumentorImpl::linkRuntime() {
}
auto InternalizeCallback = [&](Module &M, const StringSet<> &GVS) {
- internalizeModule(M, [&GVS](const GlobalValue &GV) {
- return !GV.hasName() || !GVS.count(GV.getName());
+ StringSet<> RuntimeExports;
+ for (StringRef Name : IConf.RuntimeExportSymbols->getStringList())
+ RuntimeExports.insert(Name);
+
+ internalizeModule(M, [&GVS, &RuntimeExports](const GlobalValue &GV) {
+ return !GV.hasName() || !GVS.count(GV.getName()) ||
+ RuntimeExports.count(GV.getName());
});
};
@@ -629,6 +634,17 @@ BaseConfigurationOption::createStringOption(InstrumentationConfig &IConf,
return BCO;
}
+std::unique_ptr<BaseConfigurationOption>
+BaseConfigurationOption::createStringListOption(
+ InstrumentationConfig &IConf, StringRef Name, StringRef Description,
+ ArrayRef<StringRef> DefaultValue) {
+ auto BCO = std::make_unique<BaseConfigurationOption>(Name, Description,
+ STRING_LIST);
+ BCO->setStringList(DefaultValue);
+ IConf.addBaseChoice(BCO.get());
+ return BCO;
+}
+
void InstrumentationConfig::populate(InstrumentorIRBuilderTy &IIRB) {
/// List of all instrumentation opportunities.
BasePointerIO::populate(*this, IIRB);
diff --git a/llvm/lib/Transforms/IPO/InstrumentorConfigFile.cpp b/llvm/lib/Transforms/IPO/InstrumentorConfigFile.cpp
index a2600b0551b65..083eca05b00f4 100644
--- a/llvm/lib/Transforms/IPO/InstrumentorConfigFile.cpp
+++ b/llvm/lib/Transforms/IPO/InstrumentorConfigFile.cpp
@@ -69,6 +69,12 @@ void writeConfigToJSON(InstrumentationConfig &IConf, StringRef OutputFile,
case BaseConfigurationOption::BOOLEAN:
J.attribute(BaseCO->Name, BaseCO->getBool());
break;
+ case BaseConfigurationOption::STRING_LIST:
+ J.attributeArray(BaseCO->Name, [&] {
+ for (StringRef Value : BaseCO->getStringList())
+ J.value(Value);
+ });
+ break;
}
if (!BaseCO->Description.empty())
J.attribute(std::string(BaseCO->Name) + ".description",
@@ -195,6 +201,35 @@ bool readConfigFromJSON(InstrumentationConfig &IConf, StringRef InputFile,
DS_Warning));
}
break;
+ case BaseConfigurationOption::STRING_LIST: {
+ auto *Values = ObjIt.second.getAsArray();
+ if (!Values) {
+ Ctx.diagnose(DiagnosticInfoInstrumentation(
+ Twine("configuration key '") + StringRef(ObjIt.first) +
+ Twine("' expects an array of strings, value ignored"),
+ DS_Warning));
+ break;
+ }
+
+ SmallVector<StringRef> Strings;
+ bool AllStrings = true;
+ for (const json::Value &Value : *Values) {
+ auto String = Value.getAsString();
+ if (!String) {
+ AllStrings = false;
+ break;
+ }
+ Strings.push_back(IConf.SS.save(*String));
+ }
+ if (AllStrings)
+ BO->setStringList(Strings);
+ else
+ Ctx.diagnose(DiagnosticInfoInstrumentation(
+ Twine("configuration key '") + StringRef(ObjIt.first) +
+ Twine("' expects an array of strings, value ignored"),
+ DS_Warning));
+ break;
+ }
}
} else if (!StringRef(ObjIt.first).ends_with(".description")) {
std::string Diag = "configuration key '" + ObjIt.first.str() +
diff --git a/llvm/test/Instrumentation/Instrumentor/default_config.json b/llvm/test/Instrumentation/Instrumentor/default_config.json
index fc712537e5e22..06057833ed8a6 100644
--- a/llvm/test/Instrumentation/Instrumentor/default_config.json
+++ b/llvm/test/Instrumentation/Instrumentor/default_config.json
@@ -17,7 +17,9 @@
"runtime_bitcode": "",
"runtime_bitcode.description": "Link runtime bitcode",
"inline_runtime": true,
- "inline_runtime.description": "Inline runtime function calls eagerly"
+ "inline_runtime.description": "Inline runtime function calls eagerly",
+ "runtime_export_symbols": [],
+ "runtime_export_symbols.description": "Runtime symbols that remain externally visible after linking."
},
"module_pre": {
"module": {
@@ -446,4 +448,4 @@
"id.description": "A unique ID associated with the given instrumentor call"
}
}
-}
\ No newline at end of file
+}
>From 314f775d7f726d01da9b25721739afe26c6d60c1 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Mon, 27 Jul 2026 16:48:28 -0700
Subject: [PATCH 2/9] [Instrumentor] Add runtime exports test
---
.../Instrumentor/runtime_export_symbols.ll | 14 ++++++++++++++
.../runtime_export_symbols_config.json | 15 +++++++++++++++
.../runtimes/runtime_export_symbols_rt.ll | 14 ++++++++++++++
3 files changed, 43 insertions(+)
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
new file mode 100644
index 0000000000000..5e822baa0a7a7
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
@@ -0,0 +1,14 @@
+; RUN: llvm-as %S/runtimes/runtime_export_symbols_rt.ll -o runtime_export_symbols_rt.bc
+; RUN: opt < %s -passes=instrumentor -instrumentor-read-config-files=%S/runtime_export_symbols_config.json -S | FileCheck %s
+
+; CHECK: @runtime_export = linkonce_odr global i32 0, comdat, align 4
+; CHECK: @runtime_internal = internal global i32 0, align 4
+
+ at runtime_export = external global i32
+ at runtime_internal = external global i32
+
+define i32 @test(i32 %lhs, i32 %rhs) {
+entry:
+ %result = add i32 %lhs, %rhs
+ ret i32 %result
+}
\ No newline at end of file
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json
new file mode 100644
index 0000000000000..2519dfb3796b3
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json
@@ -0,0 +1,15 @@
+{
+ "configuration": {
+ "runtime_prefix": "__runtime_export_",
+ "runtime_bitcode": "runtime_export_symbols_rt.bc",
+ "inline_runtime": false,
+ "runtime_export_symbols": ["runtime_export"]
+ },
+ "instruction_pre": {
+ "numeric": {
+ "enabled": true,
+ "size": true,
+ "id": true
+ }
+ }
+}
\ No newline at end of file
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
new file mode 100644
index 0000000000000..342dd718271b9
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
@@ -0,0 +1,14 @@
+$runtime_export = comdat any
+$runtime_internal = comdat any
+
+ at runtime_export = linkonce_odr global i32 0, comdat, align 4
+ at runtime_internal = linkonce_odr global i32 0, comdat, align 4
+
+define void @__runtime_export_pre_numeric(i32 %size, i32 %id) {
+entry:
+ %export = load i32, ptr @runtime_export, align 4
+ %internal = load i32, ptr @runtime_internal, align 4
+ %sum = add i32 %export, %internal
+ store i32 %sum, ptr @runtime_internal, align 4
+ ret void
+}
\ No newline at end of file
>From 7d319f01dbd7dab2c3ba7420e55dea99c008f534 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Fri, 14 Aug 2026 07:34:06 -0700
Subject: [PATCH 3/9] [Instrumentor] Format
---
llvm/include/llvm/Transforms/IPO/Instrumentor.h | 4 ++--
llvm/lib/Transforms/IPO/Instrumentor.cpp | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/Transforms/IPO/Instrumentor.h b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
index 6fbc620a7c4a9..915c3f3e35476 100644
--- a/llvm/include/llvm/Transforms/IPO/Instrumentor.h
+++ b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
@@ -399,8 +399,8 @@ struct LLVM_ABI InstrumentationConfig {
InlineRuntimeEagerly = BaseConfigurationOption::createBoolOption(
*this, "inline_runtime", "Inline runtime function calls eagerly", true);
RuntimeExportSymbols = BaseConfigurationOption::createStringListOption(
- *this, "runtime_export_symbols",
- "Runtime symbols that remain externally visible after linking.", {});
+ *this, "runtime_export_symbols",
+ "Runtime symbols that remain externally visible after linking.", {});
populate(IIRB);
}
diff --git a/llvm/lib/Transforms/IPO/Instrumentor.cpp b/llvm/lib/Transforms/IPO/Instrumentor.cpp
index 4bd1af2312af8..fc5b8a9845bef 100644
--- a/llvm/lib/Transforms/IPO/Instrumentor.cpp
+++ b/llvm/lib/Transforms/IPO/Instrumentor.cpp
@@ -638,8 +638,8 @@ std::unique_ptr<BaseConfigurationOption>
BaseConfigurationOption::createStringListOption(
InstrumentationConfig &IConf, StringRef Name, StringRef Description,
ArrayRef<StringRef> DefaultValue) {
- auto BCO = std::make_unique<BaseConfigurationOption>(Name, Description,
- STRING_LIST);
+ auto BCO =
+ std::make_unique<BaseConfigurationOption>(Name, Description, STRING_LIST);
BCO->setStringList(DefaultValue);
IConf.addBaseChoice(BCO.get());
return BCO;
>From 0dfbacd0bc6ce9c244e24d47497507eeb0122b72 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Fri, 14 Aug 2026 07:59:58 -0700
Subject: [PATCH 4/9] [Instrumentor] Add pre-runtime-link extension point
---
.../llvm/Transforms/IPO/Instrumentor.h | 7 ++
llvm/lib/Transforms/IPO/Instrumentor.cpp | 2 +
llvm/unittests/Transforms/IPO/CMakeLists.txt | 1 +
.../Transforms/IPO/InstrumentorTest.cpp | 94 +++++++++++++++++++
4 files changed, 104 insertions(+)
create mode 100644 llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
diff --git a/llvm/include/llvm/Transforms/IPO/Instrumentor.h b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
index 915c3f3e35476..34a4cea86d220 100644
--- a/llvm/include/llvm/Transforms/IPO/Instrumentor.h
+++ b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
@@ -407,6 +407,13 @@ struct LLVM_ABI InstrumentationConfig {
/// Populate the instrumentation opportunities.
virtual void populate(InstrumentorIRBuilderTy &IIRB);
+ /// Allow embedded users to extend the module after instrumentation and before
+ /// runtime bitcode linking.
+ virtual bool instrumentBeforeRuntimeLink(Module &,
+ InstrumentorIRBuilderTy &) {
+ return false;
+ }
+
/// Get the runtime prefix for the instrumentation runtime functions.
StringRef getRTName() const { return RuntimePrefix->getString(); }
diff --git a/llvm/lib/Transforms/IPO/Instrumentor.cpp b/llvm/lib/Transforms/IPO/Instrumentor.cpp
index fc5b8a9845bef..44e95c32e1d43 100644
--- a/llvm/lib/Transforms/IPO/Instrumentor.cpp
+++ b/llvm/lib/Transforms/IPO/Instrumentor.cpp
@@ -541,6 +541,8 @@ bool InstrumentorImpl::instrument() {
for (Function &Fn : M)
Changed |= instrumentFunction(Fn);
+ Changed |= IConf.instrumentBeforeRuntimeLink(M, IIRB);
+
linkRuntime();
return Changed;
diff --git a/llvm/unittests/Transforms/IPO/CMakeLists.txt b/llvm/unittests/Transforms/IPO/CMakeLists.txt
index 5b45191afc711..a54bf05b047b5 100644
--- a/llvm/unittests/Transforms/IPO/CMakeLists.txt
+++ b/llvm/unittests/Transforms/IPO/CMakeLists.txt
@@ -14,5 +14,6 @@ add_llvm_unittest(IPOTests
AttributorTest.cpp
FunctionSpecializationTest.cpp
ImportIDTableTests.cpp
+ InstrumentorTest.cpp
MergeFunctionsTest.cpp
)
diff --git a/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp b/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
new file mode 100644
index 0000000000000..4f89ada4dbb35
--- /dev/null
+++ b/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
@@ -0,0 +1,94 @@
+//===- InstrumentorTest.cpp - Unit tests for InstrumentorPass -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO/Instrumentor.h"
+
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/raw_ostream.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::instrumentor;
+
+namespace {
+
+class PreRuntimeLinkConfig final : public InstrumentationConfig {
+public:
+ explicit PreRuntimeLinkConfig(StringRef RuntimePath)
+ : RuntimePath(RuntimePath) {}
+
+ void populate(InstrumentorIRBuilderTy &) override {
+ RuntimeBitcode->setString(RuntimePath);
+ }
+
+ bool instrumentBeforeRuntimeLink(Module &M,
+ InstrumentorIRBuilderTy &) override {
+ LLVMContext &Ctx = M.getContext();
+ Type *Ty = Type::getInt1Ty(Ctx);
+ Constant *SawRuntime =
+ ConstantInt::get(Ty, M.getNamedGlobal("runtime_marker") != nullptr);
+ new GlobalVariable(M, Ty, false, GlobalValue::ExternalLinkage, SawRuntime,
+ "hook_saw_runtime");
+ return true;
+ }
+
+private:
+ std::string RuntimePath;
+};
+
+std::unique_ptr<Module> parseModule(StringRef IR, LLVMContext &Ctx) {
+ SMDiagnostic Err;
+ std::unique_ptr<Module> M = parseAssemblyString(IR, Err, Ctx);
+ EXPECT_TRUE(M);
+ return M;
+}
+
+TEST(InstrumentorTest, RunsHookBeforeRuntimeLink) {
+ SmallString<128> RuntimePath;
+ int FD;
+ ASSERT_FALSE(sys::fs::createTemporaryFile("instrumentor-runtime", "ll", FD,
+ RuntimePath));
+ auto RemoveRuntime = make_scope_exit([&] { sys::fs::remove(RuntimePath); });
+
+ raw_fd_ostream OS(FD, true);
+ OS << "@runtime_marker = global i32 0\n";
+ OS.close();
+
+ LLVMContext Ctx;
+ std::unique_ptr<Module> M = parseModule(R"ir(
+ define void @test() {
+ entry:
+ ret void
+ }
+ )ir",
+ Ctx);
+ ASSERT_TRUE(M);
+
+ PreRuntimeLinkConfig Config(RuntimePath);
+ ModuleAnalysisManager MAM;
+ InstrumentorPass Pass(nullptr, &Config);
+ Pass.run(*M, MAM);
+
+ GlobalVariable *HookSawRuntime = M->getNamedGlobal("hook_saw_runtime");
+ ASSERT_NE(HookSawRuntime, nullptr);
+ auto *Initializer = dyn_cast<ConstantInt>(HookSawRuntime->getInitializer());
+ ASSERT_NE(Initializer, nullptr);
+ EXPECT_TRUE(Initializer->isZero());
+ EXPECT_NE(M->getNamedGlobal("runtime_marker"), nullptr);
+}
+
+} // namespace
>From 82bd21f5137f40a59cf43752d69bc50ad17e73e5 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Fri, 14 Aug 2026 08:54:19 -0700
Subject: [PATCH 5/9] [Instrumentor] Fix test
---
llvm/unittests/Transforms/IPO/InstrumentorTest.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp b/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
index 4f89ada4dbb35..d3b9df14c0994 100644
--- a/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
+++ b/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
@@ -18,6 +18,7 @@
#include "llvm/IR/PassManager.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Support/raw_ostream.h"
#include "gtest/gtest.h"
@@ -62,7 +63,7 @@ TEST(InstrumentorTest, RunsHookBeforeRuntimeLink) {
int FD;
ASSERT_FALSE(sys::fs::createTemporaryFile("instrumentor-runtime", "ll", FD,
RuntimePath));
- auto RemoveRuntime = make_scope_exit([&] { sys::fs::remove(RuntimePath); });
+ scope_exit RemoveRuntime([&] { sys::fs::remove(RuntimePath); });
raw_fd_ostream OS(FD, true);
OS << "@runtime_marker = global i32 0\n";
@@ -80,7 +81,7 @@ TEST(InstrumentorTest, RunsHookBeforeRuntimeLink) {
PreRuntimeLinkConfig Config(RuntimePath);
ModuleAnalysisManager MAM;
- InstrumentorPass Pass(nullptr, &Config);
+ InstrumentorPass Pass(/*FS=*/nullptr, &Config, /*IIRB=*/nullptr);
Pass.run(*M, MAM);
GlobalVariable *HookSawRuntime = M->getNamedGlobal("hook_saw_runtime");
>From f5ede09f8127021a4d4977c8b8b7dfb5fa364f51 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Fri, 14 Aug 2026 10:56:08 -0700
Subject: [PATCH 6/9] [Instrumentor] Use weak linkage for runtime exports
---
llvm/lib/Transforms/IPO/Instrumentor.cpp | 7 +++++++
.../Instrumentor/runtime_export_symbols.ll | 4 ++--
.../Instrumentor/runtimes/runtime_export_symbols_rt.ll | 9 +++------
3 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/IPO/Instrumentor.cpp b/llvm/lib/Transforms/IPO/Instrumentor.cpp
index 44e95c32e1d43..13b911ea7ad9f 100644
--- a/llvm/lib/Transforms/IPO/Instrumentor.cpp
+++ b/llvm/lib/Transforms/IPO/Instrumentor.cpp
@@ -270,6 +270,13 @@ void InstrumentorImpl::linkRuntime() {
return !GV.hasName() || !GVS.count(GV.getName()) ||
RuntimeExports.count(GV.getName());
});
+
+ for (StringRef Name : IConf.RuntimeExportSymbols->getStringList()) {
+ GlobalValue *GV = M.getNamedValue(Name);
+ if (!GV || GV->isDeclarationForLinker() || GV->hasLocalLinkage())
+ continue;
+ GV->setLinkage(GlobalValue::WeakAnyLinkage);
+ }
};
if (Linker::linkModules(M, std::move(RTM), 0, InternalizeCallback)) {
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
index 5e822baa0a7a7..ad804f97ec53b 100644
--- a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
+++ b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
@@ -1,7 +1,7 @@
; RUN: llvm-as %S/runtimes/runtime_export_symbols_rt.ll -o runtime_export_symbols_rt.bc
; RUN: opt < %s -passes=instrumentor -instrumentor-read-config-files=%S/runtime_export_symbols_config.json -S | FileCheck %s
-; CHECK: @runtime_export = linkonce_odr global i32 0, comdat, align 4
+; CHECK: @runtime_export = weak global i32 0, align 4
; CHECK: @runtime_internal = internal global i32 0, align 4
@runtime_export = external global i32
@@ -11,4 +11,4 @@ define i32 @test(i32 %lhs, i32 %rhs) {
entry:
%result = add i32 %lhs, %rhs
ret i32 %result
-}
\ No newline at end of file
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
index 342dd718271b9..c13a521f24ff9 100644
--- a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
+++ b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
@@ -1,8 +1,5 @@
-$runtime_export = comdat any
-$runtime_internal = comdat any
-
- at runtime_export = linkonce_odr global i32 0, comdat, align 4
- at runtime_internal = linkonce_odr global i32 0, comdat, align 4
+ at runtime_export = global i32 0, align 4
+ at runtime_internal = global i32 0, align 4
define void @__runtime_export_pre_numeric(i32 %size, i32 %id) {
entry:
@@ -11,4 +8,4 @@ entry:
%sum = add i32 %export, %internal
store i32 %sum, ptr @runtime_internal, align 4
ret void
-}
\ No newline at end of file
+}
>From 6e4f29a196aa7591c27c9cc9173476500c9d8e17 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Wed, 19 Aug 2026 07:49:15 -0700
Subject: [PATCH 7/9] [Instrumentor] Support multiple runtime bitcodes
---
.../llvm/Transforms/IPO/Instrumentor.h | 10 ++--
llvm/lib/Transforms/IPO/Instrumentor.cpp | 52 +++++++------------
.../Instrumentor/default_config.json | 8 ++-
.../Instrumentor/inline_runtime_config.json | 2 +-
.../Instrumentor/runtime_bitcodes.ll | 17 ++++++
.../Instrumentor/runtime_bitcodes_config.json | 17 ++++++
.../Instrumentor/runtime_export_symbols.ll | 14 -----
.../runtime_export_symbols_config.json | 15 ------
.../runtimes/runtime_bitcodes_callbacks_rt.ll | 11 ++++
.../runtimes/runtime_bitcodes_state_rt.ll | 1 +
.../runtimes/runtime_export_symbols_rt.ll | 11 ----
.../Transforms/IPO/InstrumentorTest.cpp | 2 +-
12 files changed, 74 insertions(+), 86 deletions(-)
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtime_bitcodes.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtime_bitcodes_config.json
delete mode 100644 llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
delete mode 100644 llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_callbacks_rt.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_state_rt.ll
delete mode 100644 llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
diff --git a/llvm/include/llvm/Transforms/IPO/Instrumentor.h b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
index 34a4cea86d220..202b132ec3962 100644
--- a/llvm/include/llvm/Transforms/IPO/Instrumentor.h
+++ b/llvm/include/llvm/Transforms/IPO/Instrumentor.h
@@ -394,13 +394,10 @@ struct LLVM_ABI InstrumentationConfig {
*this, "host_enabled", "Instrument non-GPU targets", true);
GPUEnabled = BaseConfigurationOption::createBoolOption(
*this, "gpu_enabled", "Instrument GPU targets", true);
- RuntimeBitcode = BaseConfigurationOption::createStringOption(
- *this, "runtime_bitcode", "Link runtime bitcode", "");
+ RuntimeBitcodes = BaseConfigurationOption::createStringListOption(
+ *this, "runtime_bitcodes", "Link runtime bitcode files", {});
InlineRuntimeEagerly = BaseConfigurationOption::createBoolOption(
*this, "inline_runtime", "Inline runtime function calls eagerly", true);
- RuntimeExportSymbols = BaseConfigurationOption::createStringListOption(
- *this, "runtime_export_symbols",
- "Runtime symbols that remain externally visible after linking.", {});
populate(IIRB);
}
@@ -479,9 +476,8 @@ struct LLVM_ABI InstrumentationConfig {
std::unique_ptr<BaseConfigurationOption> FunctionRegex;
std::unique_ptr<BaseConfigurationOption> HostEnabled;
std::unique_ptr<BaseConfigurationOption> GPUEnabled;
- std::unique_ptr<BaseConfigurationOption> RuntimeBitcode;
+ std::unique_ptr<BaseConfigurationOption> RuntimeBitcodes;
std::unique_ptr<BaseConfigurationOption> InlineRuntimeEagerly;
- std::unique_ptr<BaseConfigurationOption> RuntimeExportSymbols;
/// The map registered instrumentation opportunities. The map is indexed by
/// the instrumentation location kind and then by the opportunity name. Notice
diff --git a/llvm/lib/Transforms/IPO/Instrumentor.cpp b/llvm/lib/Transforms/IPO/Instrumentor.cpp
index 13b911ea7ad9f..f16ee0c23d082 100644
--- a/llvm/lib/Transforms/IPO/Instrumentor.cpp
+++ b/llvm/lib/Transforms/IPO/Instrumentor.cpp
@@ -50,7 +50,6 @@
#include "llvm/Support/Regex.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Transforms/IPO/InstrumentorUtils.h"
-#include "llvm/Transforms/IPO/Internalize.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include "llvm/Transforms/Utils/PromoteMemToReg.h"
@@ -247,42 +246,31 @@ static Regex createRegex(StringRef Str, StringRef Name, LLVMContext &Ctx) {
}
void InstrumentorImpl::linkRuntime() {
- const auto RuntimeBitcode = IConf.RuntimeBitcode->getString();
- if (RuntimeBitcode.empty())
+ ArrayRef<StringRef> RuntimeBitcodes = IConf.RuntimeBitcodes->getStringList();
+ if (RuntimeBitcodes.empty())
return;
- SMDiagnostic Err;
- auto RTM = parseIRFile(RuntimeBitcode, Err, M.getContext());
- if (!RTM) {
- IIRB.Ctx.diagnose(DiagnosticInfoInstrumentation(
- Twine("Failed to parse runtime bitcode file '") + RuntimeBitcode +
- Twine("':\n") + M.getName(),
- DS_Error));
- return;
- }
-
- auto InternalizeCallback = [&](Module &M, const StringSet<> &GVS) {
- StringSet<> RuntimeExports;
- for (StringRef Name : IConf.RuntimeExportSymbols->getStringList())
- RuntimeExports.insert(Name);
-
- internalizeModule(M, [&GVS, &RuntimeExports](const GlobalValue &GV) {
- return !GV.hasName() || !GVS.count(GV.getName()) ||
- RuntimeExports.count(GV.getName());
- });
+ for (StringRef RuntimeBitcode : RuntimeBitcodes) {
+ if (RuntimeBitcode.empty())
+ continue;
- for (StringRef Name : IConf.RuntimeExportSymbols->getStringList()) {
- GlobalValue *GV = M.getNamedValue(Name);
- if (!GV || GV->isDeclarationForLinker() || GV->hasLocalLinkage())
- continue;
- GV->setLinkage(GlobalValue::WeakAnyLinkage);
+ SMDiagnostic Err;
+ auto RTM = parseIRFile(RuntimeBitcode, Err, M.getContext());
+ if (!RTM) {
+ IIRB.Ctx.diagnose(DiagnosticInfoInstrumentation(
+ Twine("Failed to parse runtime bitcode file '") + RuntimeBitcode +
+ Twine("':\n") + M.getName(),
+ DS_Error));
+ return;
}
- };
- if (Linker::linkModules(M, std::move(RTM), 0, InternalizeCallback)) {
- IIRB.Ctx.diagnose(DiagnosticInfoInstrumentation(
- "Failed to link in runtime bitcode", DS_Error));
- return;
+ if (Linker::linkModules(M, std::move(RTM))) {
+ IIRB.Ctx.diagnose(DiagnosticInfoInstrumentation(
+ Twine("Failed to link in runtime bitcode file '") + RuntimeBitcode +
+ "'",
+ DS_Error));
+ return;
+ }
}
if (!IConf.InlineRuntimeEagerly->getBool())
diff --git a/llvm/test/Instrumentation/Instrumentor/default_config.json b/llvm/test/Instrumentation/Instrumentor/default_config.json
index 06057833ed8a6..362e866f7882e 100644
--- a/llvm/test/Instrumentation/Instrumentor/default_config.json
+++ b/llvm/test/Instrumentation/Instrumentor/default_config.json
@@ -14,12 +14,10 @@
"host_enabled.description": "Instrument non-GPU targets",
"gpu_enabled": true,
"gpu_enabled.description": "Instrument GPU targets",
- "runtime_bitcode": "",
- "runtime_bitcode.description": "Link runtime bitcode",
+ "runtime_bitcodes": [],
+ "runtime_bitcodes.description": "Link runtime bitcode files",
"inline_runtime": true,
- "inline_runtime.description": "Inline runtime function calls eagerly",
- "runtime_export_symbols": [],
- "runtime_export_symbols.description": "Runtime symbols that remain externally visible after linking."
+ "inline_runtime.description": "Inline runtime function calls eagerly"
},
"module_pre": {
"module": {
diff --git a/llvm/test/Instrumentation/Instrumentor/inline_runtime_config.json b/llvm/test/Instrumentation/Instrumentor/inline_runtime_config.json
index 3f459240deacd..9f406c3be2878 100644
--- a/llvm/test/Instrumentation/Instrumentor/inline_runtime_config.json
+++ b/llvm/test/Instrumentation/Instrumentor/inline_runtime_config.json
@@ -1,7 +1,7 @@
{
"configuration": {
"runtime_prefix": "__bytes_computed_",
- "runtime_bitcode": "bytes_computed_rt.bc",
+ "runtime_bitcodes": ["bytes_computed_rt.bc"],
"inline_runtime": true
},
"instruction_pre": {
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_bitcodes.ll b/llvm/test/Instrumentation/Instrumentor/runtime_bitcodes.ll
new file mode 100644
index 0000000000000..14b2cfcdcbcf2
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtime_bitcodes.ll
@@ -0,0 +1,17 @@
+; This test checks that Instrumentor links multiple runtime bitcode modules.
+; The callback runtime uses state from the preceding runtime module.
+; RUN: llvm-as %S/runtimes/runtime_bitcodes_state_rt.ll -o runtime_bitcodes_state_rt.bc
+; RUN: llvm-as %S/runtimes/runtime_bitcodes_callbacks_rt.ll -o runtime_bitcodes_callbacks_rt.bc
+; RUN: opt < %s -passes=instrumentor -instrumentor-read-config-files=%S/runtime_bitcodes_config.json -S | FileCheck %s
+
+; CHECK-DAG: @runtime_state = protected global i32 0, align 4
+; CHECK-DAG: @runtime_private_state = internal global i32 0, align 4
+; CHECK-DAG: define protected void @__runtime_bitcodes_pre_numeric(
+
+ at runtime_state = external global i32
+
+define i32 @test(i32 %lhs, i32 %rhs) {
+entry:
+ %result = add i32 %lhs, %rhs
+ ret i32 %result
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_bitcodes_config.json b/llvm/test/Instrumentation/Instrumentor/runtime_bitcodes_config.json
new file mode 100644
index 0000000000000..822f4f6b04ea2
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtime_bitcodes_config.json
@@ -0,0 +1,17 @@
+{
+ "configuration": {
+ "runtime_prefix": "__runtime_bitcodes_",
+ "runtime_bitcodes": [
+ "runtime_bitcodes_state_rt.bc",
+ "runtime_bitcodes_callbacks_rt.bc"
+ ],
+ "inline_runtime": false
+ },
+ "instruction_pre": {
+ "numeric": {
+ "enabled": true,
+ "size": true,
+ "id": true
+ }
+ }
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
deleted file mode 100644
index ad804f97ec53b..0000000000000
--- a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols.ll
+++ /dev/null
@@ -1,14 +0,0 @@
-; RUN: llvm-as %S/runtimes/runtime_export_symbols_rt.ll -o runtime_export_symbols_rt.bc
-; RUN: opt < %s -passes=instrumentor -instrumentor-read-config-files=%S/runtime_export_symbols_config.json -S | FileCheck %s
-
-; CHECK: @runtime_export = weak global i32 0, align 4
-; CHECK: @runtime_internal = internal global i32 0, align 4
-
- at runtime_export = external global i32
- at runtime_internal = external global i32
-
-define i32 @test(i32 %lhs, i32 %rhs) {
-entry:
- %result = add i32 %lhs, %rhs
- ret i32 %result
-}
diff --git a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json b/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json
deleted file mode 100644
index 2519dfb3796b3..0000000000000
--- a/llvm/test/Instrumentation/Instrumentor/runtime_export_symbols_config.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "configuration": {
- "runtime_prefix": "__runtime_export_",
- "runtime_bitcode": "runtime_export_symbols_rt.bc",
- "inline_runtime": false,
- "runtime_export_symbols": ["runtime_export"]
- },
- "instruction_pre": {
- "numeric": {
- "enabled": true,
- "size": true,
- "id": true
- }
- }
-}
\ No newline at end of file
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_callbacks_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_callbacks_rt.ll
new file mode 100644
index 0000000000000..697f7be6b79bb
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_callbacks_rt.ll
@@ -0,0 +1,11 @@
+ at runtime_state = external global i32
+ at runtime_private_state = internal global i32 0, align 4
+
+define protected void @__runtime_bitcodes_pre_numeric(i32 %size, i32 %id) {
+entry:
+ %state = load i32, ptr @runtime_state, align 4
+ %private_state = load i32, ptr @runtime_private_state, align 4
+ %sum = add i32 %state, %private_state
+ store i32 %sum, ptr @runtime_private_state, align 4
+ ret void
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_state_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_state_rt.ll
new file mode 100644
index 0000000000000..c11f53367a47a
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_bitcodes_state_rt.ll
@@ -0,0 +1 @@
+ at runtime_state = protected global i32 0, align 4
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
deleted file mode 100644
index c13a521f24ff9..0000000000000
--- a/llvm/test/Instrumentation/Instrumentor/runtimes/runtime_export_symbols_rt.ll
+++ /dev/null
@@ -1,11 +0,0 @@
- at runtime_export = global i32 0, align 4
- at runtime_internal = global i32 0, align 4
-
-define void @__runtime_export_pre_numeric(i32 %size, i32 %id) {
-entry:
- %export = load i32, ptr @runtime_export, align 4
- %internal = load i32, ptr @runtime_internal, align 4
- %sum = add i32 %export, %internal
- store i32 %sum, ptr @runtime_internal, align 4
- ret void
-}
diff --git a/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp b/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
index d3b9df14c0994..504f68214bda1 100644
--- a/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
+++ b/llvm/unittests/Transforms/IPO/InstrumentorTest.cpp
@@ -33,7 +33,7 @@ class PreRuntimeLinkConfig final : public InstrumentationConfig {
: RuntimePath(RuntimePath) {}
void populate(InstrumentorIRBuilderTy &) override {
- RuntimeBitcode->setString(RuntimePath);
+ RuntimeBitcodes->setStringList({StringRef(RuntimePath)});
}
bool instrumentBeforeRuntimeLink(Module &M,
>From 08b687de3331a4f306d05a2f669b9c46f6b9523f Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Wed, 19 Aug 2026 15:53:24 -0700
Subject: [PATCH 8/9] [Offload][InputGenGPU] Add GPU instrumentation pass
policy
---
.../include/llvm/Transforms/IPO/InputGenGPU.h | 24 +++++++
llvm/lib/Passes/PassBuilder.cpp | 1 +
llvm/lib/Passes/PassRegistry.def | 1 +
llvm/lib/Transforms/IPO/CMakeLists.txt | 1 +
llvm/lib/Transforms/IPO/InputGenGPU.cpp | 72 +++++++++++++++++++
.../Instrumentor/inputgen_gpu_host.ll | 17 +++++
.../Instrumentor/inputgen_gpu_post_load.ll | 19 +++++
.../Instrumentor/inputgen_gpu_runtime_link.ll | 16 +++++
.../runtimes/inputgen_gpu_callbacks_rt.ll | 8 +++
9 files changed, 159 insertions(+)
create mode 100644 llvm/include/llvm/Transforms/IPO/InputGenGPU.h
create mode 100644 llvm/lib/Transforms/IPO/InputGenGPU.cpp
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_host.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_post_load.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_runtime_link.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_callbacks_rt.ll
diff --git a/llvm/include/llvm/Transforms/IPO/InputGenGPU.h b/llvm/include/llvm/Transforms/IPO/InputGenGPU.h
new file mode 100644
index 0000000000000..0e9af03fb7f81
--- /dev/null
+++ b/llvm/include/llvm/Transforms/IPO/InputGenGPU.h
@@ -0,0 +1,24 @@
+//===-- InputGenGPU.h - InputGen GPU instrumentation pass -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_IPO_INPUTGENGPU_H
+#define LLVM_TRANSFORMS_IPO_INPUTGENGPU_H
+
+#include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
+
+namespace llvm {
+
+class InputGenGPUPass : public RequiredPassInfoMixin<InputGenGPUPass> {
+public:
+ LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_IPO_INPUTGENGPU_H
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 6abaa77f9065f..54e4394713b59 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -241,6 +241,7 @@
#include "llvm/Transforms/IPO/GlobalSplit.h"
#include "llvm/Transforms/IPO/HotColdSplitting.h"
#include "llvm/Transforms/IPO/InferFunctionAttrs.h"
+#include "llvm/Transforms/IPO/InputGenGPU.h"
#include "llvm/Transforms/IPO/Instrumentor.h"
#include "llvm/Transforms/IPO/Internalize.h"
#include "llvm/Transforms/IPO/LowerTypeTests.h"
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 177d8ecd3508d..c838cca01ecee 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -95,6 +95,7 @@ MODULE_PASS("hipstdpar-select-accelerator-code",
HipStdParAcceleratorCodeSelectionPass())
MODULE_PASS("hotcoldsplit", HotColdSplittingPass())
MODULE_PASS("inferattrs", InferFunctionAttrsPass())
+MODULE_PASS("inputgen-gpu", InputGenGPUPass())
MODULE_PASS("inliner-ml-advisor-release",
ModuleInlinerWrapperPass(getInlineParams(), true, {},
InliningAdvisorMode::Release, 0))
diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt
index ca0e140264829..4871873cc1f3b 100644
--- a/llvm/lib/Transforms/IPO/CMakeLists.txt
+++ b/llvm/lib/Transforms/IPO/CMakeLists.txt
@@ -24,6 +24,7 @@ add_llvm_component_library(LLVMipo
GlobalSplit.cpp
HotColdSplitting.cpp
IPO.cpp
+ InputGenGPU.cpp
InferFunctionAttrs.cpp
Inliner.cpp
Instrumentor.cpp
diff --git a/llvm/lib/Transforms/IPO/InputGenGPU.cpp b/llvm/lib/Transforms/IPO/InputGenGPU.cpp
new file mode 100644
index 0000000000000..59ab811db85f8
--- /dev/null
+++ b/llvm/lib/Transforms/IPO/InputGenGPU.cpp
@@ -0,0 +1,72 @@
+//===-- InputGenGPU.cpp - InputGen GPU instrumentation pass ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO/InputGenGPU.h"
+#include "llvm/Transforms/IPO/Instrumentor.h"
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+using namespace llvm;
+using namespace llvm::instrumentor;
+
+#define DEBUG_TYPE "inputgen-gpu"
+
+static cl::list<std::string> InputGenGPURuntimeBitcodes(
+ "inputgen-gpu-runtime-bitcode",
+ cl::desc("InputGen GPU runtime bitcode file; may be repeated"),
+ cl::ZeroOrMore);
+
+namespace {
+
+class InputGenGPUConfig final : public InstrumentationConfig {
+ void populate(InstrumentorIRBuilderTy &IIRB) override {
+ InstrumentationConfig::populate(IIRB);
+
+ RuntimePrefix->setString("__ig_");
+ HostEnabled->setBool(false);
+ GPUEnabled->setBool(true);
+ SmallVector<StringRef> RuntimeBitcodeRefs;
+ for (StringRef RuntimeBitcode : InputGenGPURuntimeBitcodes)
+ RuntimeBitcodeRefs.push_back(RuntimeBitcode);
+ RuntimeBitcodes->setStringList(RuntimeBitcodeRefs);
+ InlineRuntimeEagerly->setBool(false);
+
+ for (auto &ChoiceMap : IChoices) {
+ for (auto &ChoiceIt : ChoiceMap) {
+ auto *IO = ChoiceIt.second;
+ IO->Enabled = false;
+ IO->Filter = "";
+ for (IRTArg &Arg : IO->IRTArgs)
+ Arg.Enabled = false;
+ }
+ }
+
+ auto *PostLoad =
+ IChoices[InstrumentationLocation::INSTRUCTION_POST].lookup("load");
+ if (!PostLoad)
+ return;
+
+ PostLoad->Enabled = true;
+ for (IRTArg &Arg : PostLoad->IRTArgs) {
+ Arg.Enabled = Arg.Name == "value" || Arg.Name == "value_size" ||
+ Arg.Name == "value_type_id" || Arg.Name == "id";
+ }
+ }
+};
+
+} // end anonymous namespace
+
+PreservedAnalyses InputGenGPUPass::run(Module &M, ModuleAnalysisManager &MAM) {
+ InputGenGPUConfig IConf;
+ InstrumentorIRBuilderTy IIRB(M);
+ return InstrumentorPass(/*FS=*/nullptr, &IConf, &IIRB).run(M, MAM);
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_host.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_host.ll
new file mode 100644
index 0000000000000..dfef0a5f356df
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_host.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -passes=inputgen-gpu -S | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; The inputgen-gpu pass is GPU-only, so host IR should remain unchanged.
+; CHECK-LABEL: define i32 @vvv_foo(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %v = load i32, ptr %a, align 4
+; CHECK-NEXT: ret i32 %v
+; CHECK-NEXT: }
+
+define i32 @vvv_foo(ptr noundef %a) {
+entry:
+ %v = load i32, ptr %a, align 4
+ ret i32 %v
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_post_load.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_post_load.ll
new file mode 100644
index 0000000000000..bba0ef436df30
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_post_load.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -passes=inputgen-gpu -S | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+; CHECK-NOT: __ig_pre_load
+; CHECK-NOT: __ig_post_store
+; CHECK-LABEL: define hidden i32 @vvv_foo(
+; CHECK: [[LOAD:%.*]] = load i32, ptr {{%.*}}, align 4
+; CHECK: [[EXT:%.*]] = zext i32 [[LOAD]] to i64
+; CHECK: [[CALL:%.*]] = call i64 @__ig_post_load(i64 [[EXT]], i64 4, i32 12, i32 -1)
+; CHECK: [[TRUNC:%.*]] = trunc i64 [[CALL]] to i32
+; CHECK: ret i32 [[TRUNC]]
+
+define hidden i32 @vvv_foo(ptr noundef %a) {
+entry:
+ %v = load i32, ptr %a, align 4
+ ret i32 %v
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_runtime_link.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_runtime_link.ll
new file mode 100644
index 0000000000000..47478dae9bd74
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_runtime_link.ll
@@ -0,0 +1,16 @@
+; RUN: llvm-as %S/runtimes/inputgen_gpu_callbacks_rt.ll -o %t.rt.bc
+; RUN: opt < %s -passes=inputgen-gpu -inputgen-gpu-runtime-bitcode=%t.rt.bc -S | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+; CHECK-NOT: @inputgen_
+; CHECK-LABEL: define hidden i32 @vvv_foo(
+; CHECK: call i64 @__ig_post_load(
+; CHECK-LABEL: define protected i64 @__ig_post_load(
+
+define hidden i32 @vvv_foo(ptr noundef %a) {
+entry:
+ %v = load i32, ptr %a, align 4
+ ret i32 %v
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_callbacks_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_callbacks_rt.ll
new file mode 100644
index 0000000000000..8d5e27b1bdf53
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_callbacks_rt.ll
@@ -0,0 +1,8 @@
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+define protected i64 @__ig_post_load(i64 %value, i64 %value_size,
+ i32 %value_type_id, i32 %id) {
+entry:
+ ret i64 %value
+}
>From 3f62050459888fbf9292e21acdf8774f4abb8874 Mon Sep 17 00:00:00 2001
From: "Vincent A. Arcila Larrea" <arcilalarrea1 at llnl.gov>
Date: Fri, 14 Aug 2026 10:10:34 -0700
Subject: [PATCH 9/9] [Offload][InputGenGPU] Generate GPU direct-entry kernels
---
.../Frontend/Offloading/InputGenGPUABI.def | 36 +++++
llvm/lib/Transforms/IPO/InputGenGPU.cpp | 147 ++++++++++++++++++
.../inputgen_gpu_duplicate_entry.ll | 17 ++
.../Instrumentor/inputgen_gpu_entry.ll | 33 ++++
.../Instrumentor/inputgen_gpu_host_entry.ll | 16 ++
.../inputgen_gpu_missing_entry.ll | 16 ++
.../Instrumentor/inputgen_gpu_no_entry.ll | 14 ++
.../Instrumentor/inputgen_gpu_nvptx_entry.ll | 11 ++
.../Instrumentor/inputgen_gpu_runtime_link.ll | 16 +-
.../Instrumentor/inputgen_gpu_scalar_args.ll | 15 ++
.../Instrumentor/inputgen_gpu_void_return.ll | 14 ++
.../inputgen_gpu_entry_callbacks_rt.ll | 13 ++
.../runtimes/inputgen_gpu_entry_state_rt.ll | 7 +
13 files changed, 350 insertions(+), 5 deletions(-)
create mode 100644 llvm/include/llvm/Frontend/Offloading/InputGenGPUABI.def
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_duplicate_entry.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_entry.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_host_entry.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_missing_entry.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_no_entry.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_nvptx_entry.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_scalar_args.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/inputgen_gpu_void_return.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_entry_callbacks_rt.ll
create mode 100644 llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_entry_state_rt.ll
diff --git a/llvm/include/llvm/Frontend/Offloading/InputGenGPUABI.def b/llvm/include/llvm/Frontend/Offloading/InputGenGPUABI.def
new file mode 100644
index 0000000000000..5edec60803403
--- /dev/null
+++ b/llvm/include/llvm/Frontend/Offloading/InputGenGPUABI.def
@@ -0,0 +1,36 @@
+//===-- InputGen GPU direct-entry ABI definitions -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Define INPUTGEN_GPU_ENTRY_STATE and/or INPUTGEN_GPU_ABI_MODE before including
+// this file to generate declarations for the direct-entry ABI.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef INPUTGEN_GPU_ENTRY_STATE
+#define INPUTGEN_GPU_ENTRY_STATE(Variable, Constant, CType, Symbol)
+#endif
+
+#ifndef INPUTGEN_GPU_ABI_MODE
+#define INPUTGEN_GPU_ABI_MODE(Name, Value)
+#endif
+
+INPUTGEN_GPU_ENTRY_STATE(InputGenEntryBuffer, InputGenEntryBufferSymbol, int *,
+ "inputgen_buffer")
+INPUTGEN_GPU_ENTRY_STATE(InputGenEntryBufferSize, InputGenEntryBufferSizeSymbol,
+ uint64_t, "inputgen_buffer_size")
+INPUTGEN_GPU_ENTRY_STATE(InputGenEntryBufferOffset,
+ InputGenEntryBufferOffsetSymbol, uint64_t,
+ "inputgen_buffer_offset")
+INPUTGEN_GPU_ENTRY_STATE(InputGenEntryMode, InputGenEntryModeSymbol, int,
+ "inputgen_mode")
+
+INPUTGEN_GPU_ABI_MODE(INPUTGEN_MODE_GENERATE, 1)
+INPUTGEN_GPU_ABI_MODE(INPUTGEN_MODE_REPLAY, 2)
+
+#undef INPUTGEN_GPU_ENTRY_STATE
+#undef INPUTGEN_GPU_ABI_MODE
diff --git a/llvm/lib/Transforms/IPO/InputGenGPU.cpp b/llvm/lib/Transforms/IPO/InputGenGPU.cpp
index 59ab811db85f8..5670e421b6fd8 100644
--- a/llvm/lib/Transforms/IPO/InputGenGPU.cpp
+++ b/llvm/lib/Transforms/IPO/InputGenGPU.cpp
@@ -11,15 +11,32 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/CallingConv.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/Instructions.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/VirtualFileSystem.h"
+#include "llvm/TargetParser/Triple.h"
+#include "llvm/Transforms/Utils/Instrumentation.h"
using namespace llvm;
using namespace llvm::instrumentor;
#define DEBUG_TYPE "inputgen-gpu"
+static cl::opt<std::string> InputGenGPUEntryFunction(
+ "inputgen-gpu-entry-function",
+ cl::desc("Device function wrapped by the InputGen GPU entry kernel"),
+ cl::init(""));
+
static cl::list<std::string> InputGenGPURuntimeBitcodes(
"inputgen-gpu-runtime-bitcode",
cl::desc("InputGen GPU runtime bitcode file; may be repeated"),
@@ -27,6 +44,122 @@ static cl::list<std::string> InputGenGPURuntimeBitcodes(
namespace {
+#define INPUTGEN_GPU_ENTRY_STATE(Variable, Constant, CType, Symbol) \
+ constexpr StringLiteral Constant(Symbol);
+#include "llvm/Frontend/Offloading/InputGenGPUABI.def"
+
+std::string getInputGenGPUEntryPointName(StringRef EntryFunctionName) {
+ return (Twine("__ig_entry_") + EntryFunctionName).str();
+}
+
+bool createInputGenGPUEntryKernel(Module &M, InstrumentorIRBuilderTy &IIRB,
+ StringRef EntryFunctionName) {
+ if (EntryFunctionName.empty())
+ return false;
+
+ Function *EntryFn = M.getFunction(EntryFunctionName);
+ if (!EntryFn || EntryFn->isDeclaration()) {
+ IIRB.Ctx.diagnose(DiagnosticInfoInstrumentation(
+ Twine("inputgen entry function '") + EntryFunctionName +
+ "' was not found or is only a declaration",
+ DS_Warning));
+ return false;
+ }
+
+ CallingConv::ID KernelCC;
+ const Triple &T = M.getTargetTriple();
+ if (T.isAMDGPU())
+ KernelCC = CallingConv::AMDGPU_KERNEL;
+ else if (T.isNVPTX())
+ KernelCC = CallingConv::PTX_Kernel;
+ else {
+ IIRB.Ctx.diagnose(DiagnosticInfoInstrumentation(
+ Twine("inputgen entry kernels are not supported for target '") +
+ T.str() + "'",
+ DS_Warning));
+ return false;
+ }
+
+ std::string EntryPointName = getInputGenGPUEntryPointName(EntryFunctionName);
+ if (M.getNamedValue(EntryPointName)) {
+ IIRB.Ctx.diagnose(DiagnosticInfoInstrumentation(
+ Twine("inputgen entry point '") + EntryPointName + "' already exists",
+ DS_Warning));
+ return false;
+ }
+
+ const DataLayout &DL = M.getDataLayout();
+ unsigned GlobalAS = DL.getDefaultGlobalsAddressSpace();
+ auto GetOrInsertGlobalInDefaultAS = [&](StringRef Name, Type *Ty) {
+ return M.getOrInsertGlobal(Name, Ty, [&] {
+ return new GlobalVariable(
+ M, Ty, /*isConstant=*/false, GlobalValue::ExternalLinkage,
+ /*Initializer=*/nullptr, Name,
+ /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal, GlobalAS);
+ });
+ };
+
+ GlobalVariable *BufferGV =
+ GetOrInsertGlobalInDefaultAS(InputGenEntryBufferSymbol, IIRB.PtrTy);
+ GlobalVariable *BufferSizeGV =
+ GetOrInsertGlobalInDefaultAS(InputGenEntryBufferSizeSymbol, IIRB.Int64Ty);
+ GlobalVariable *BufferOffsetGV = GetOrInsertGlobalInDefaultAS(
+ InputGenEntryBufferOffsetSymbol, IIRB.Int64Ty);
+ GlobalVariable *ModeGV =
+ GetOrInsertGlobalInDefaultAS(InputGenEntryModeSymbol, IIRB.Int32Ty);
+
+ FunctionType *EntryPointTy = FunctionType::get(
+ IIRB.VoidTy, {IIRB.Int32Ty, IIRB.PtrTy, IIRB.Int64Ty, IIRB.PtrTy},
+ /*isVarArg=*/false);
+ Function *EntryPoint = Function::Create(
+ EntryPointTy, GlobalValue::ExternalLinkage, EntryPointName, M);
+ EntryPoint->setCallingConv(KernelCC);
+
+ auto ArgIt = EntryPoint->arg_begin();
+ Argument *Mode = &*ArgIt++;
+ Argument *Buffer = &*ArgIt++;
+ Argument *Size = &*ArgIt++;
+ Argument *Result = &*ArgIt++;
+ Mode->setName("mode");
+ Buffer->setName("input_buffer");
+ Size->setName("input_buffer_size");
+ Result->setName("result");
+
+ BasicBlock *EntryBB = BasicBlock::Create(IIRB.Ctx, "entry", EntryPoint);
+ IIRB.IRB.SetInsertPoint(EntryBB);
+
+ IIRB.IRB.CreateAlignedStore(Mode, ModeGV, DL.getABITypeAlign(IIRB.Int32Ty));
+ IIRB.IRB.CreateAlignedStore(Buffer, BufferGV, DL.getABITypeAlign(IIRB.PtrTy));
+ IIRB.IRB.CreateAlignedStore(Size, BufferSizeGV,
+ DL.getABITypeAlign(IIRB.Int64Ty));
+ IIRB.IRB.CreateAlignedStore(ConstantInt::get(IIRB.Int64Ty, 0), BufferOffsetGV,
+ DL.getABITypeAlign(IIRB.Int64Ty));
+
+ SmallVector<Value *> Args;
+ Args.reserve(EntryFn->arg_size());
+ for (Argument &Arg : EntryFn->args()) {
+ Type *ArgTy = Arg.getType();
+ if (!ArgTy->isPointerTy()) {
+ Args.push_back(Constant::getNullValue(ArgTy));
+ continue;
+ }
+
+ // Create placeholder storage for pointer arguments.
+ AllocaInst *AI =
+ IIRB.IRB.CreateAlloca(IIRB.Int64Ty, DL.getAllocaAddrSpace());
+ AI->setAlignment(Align(8));
+ Args.push_back(IIRB.IRB.CreatePointerBitCastOrAddrSpaceCast(AI, ArgTy));
+ }
+
+ CallInst *CI = IIRB.IRB.CreateCall(EntryFn->getFunctionType(), EntryFn, Args);
+ if (!EntryFn->getReturnType()->isVoidTy())
+ IIRB.IRB.CreateAlignedStore(CI, Result,
+ DL.getABITypeAlign(EntryFn->getReturnType()));
+ IIRB.IRB.CreateRetVoid();
+
+ return true;
+}
+
class InputGenGPUConfig final : public InstrumentationConfig {
void populate(InstrumentorIRBuilderTy &IIRB) override {
InstrumentationConfig::populate(IIRB);
@@ -61,11 +194,25 @@ class InputGenGPUConfig final : public InstrumentationConfig {
Arg.Name == "value_type_id" || Arg.Name == "id";
}
}
+
+ bool instrumentBeforeRuntimeLink(Module &M,
+ InstrumentorIRBuilderTy &IIRB) override {
+ return createInputGenGPUEntryKernel(M, IIRB, InputGenGPUEntryFunction);
+ }
};
} // end anonymous namespace
PreservedAnalyses InputGenGPUPass::run(Module &M, ModuleAnalysisManager &MAM) {
+ const Triple &T = M.getTargetTriple();
+ if (!InputGenGPUEntryFunction.empty() && !T.isAMDGPU() && !T.isNVPTX()) {
+ M.getContext().diagnose(DiagnosticInfoInstrumentation(
+ Twine("inputgen entry kernels are not supported for target '") +
+ T.str() + "'",
+ DS_Warning));
+ return PreservedAnalyses::all();
+ }
+
InputGenGPUConfig IConf;
InstrumentorIRBuilderTy IIRB(M);
return InstrumentorPass(/*FS=*/nullptr, &IConf, &IIRB).run(M, MAM);
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_duplicate_entry.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_duplicate_entry.ll
new file mode 100644
index 0000000000000..812b92449f980
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_duplicate_entry.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -passes=inputgen-gpu -inputgen-gpu-entry-function=vvv_foo -disable-output 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+; CHECK: inputgen entry point '__ig_entry_vvv_foo' already exists
+
+define hidden i32 @vvv_foo(ptr noundef %a) {
+entry:
+ %v = load i32, ptr %a, align 4
+ ret i32 %v
+}
+
+define amdgpu_kernel void @__ig_entry_vvv_foo() {
+entry:
+ ret void
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_entry.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_entry.ll
new file mode 100644
index 0000000000000..ca42f2637b60b
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_entry.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -passes=inputgen-gpu -inputgen-gpu-entry-function=vvv_foo -S | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+; CHECK: @inputgen_buffer = external addrspace(1) global ptr
+; CHECK: @inputgen_buffer_size = external addrspace(1) global i64
+; CHECK: @inputgen_buffer_offset = external addrspace(1) global i64
+; CHECK: @inputgen_mode = external addrspace(1) global i32
+
+define hidden i32 @vvv_foo(ptr noundef %a) {
+; CHECK-LABEL: define hidden i32 @vvv_foo(
+; CHECK: [[LOAD:%.*]] = load i32, ptr {{%.*}}, align 4
+; CHECK: [[EXT:%.*]] = zext i32 [[LOAD]] to i64
+; CHECK: [[CALL:%.*]] = call i64 @__ig_post_load(i64 [[EXT]], i64 4, i32 12, i32 -1)
+; CHECK: [[TRUNC:%.*]] = trunc i64 [[CALL]] to i32
+; CHECK: ret i32 [[TRUNC]]
+entry:
+ %v = load i32, ptr %a, align 4
+ ret i32 %v
+}
+
+; CHECK-LABEL: define amdgpu_kernel void @__ig_entry_vvv_foo(
+; CHECK-SAME: i32 [[MODE:%.*]], ptr [[BUFFER:%.*]], i64 [[SIZE:%.*]], ptr [[RESULT:%.*]]) {
+; CHECK: store i32 [[MODE]], ptr addrspace(1) @inputgen_mode, align 4
+; CHECK: store ptr [[BUFFER]], ptr addrspace(1) @inputgen_buffer, align 8
+; CHECK: store i64 [[SIZE]], ptr addrspace(1) @inputgen_buffer_size, align 8
+; CHECK: store i64 0, ptr addrspace(1) @inputgen_buffer_offset, align 8
+; CHECK: [[ARG:%.*]] = alloca i64, align 8, addrspace(5)
+; CHECK: [[ARG_CAST:%.*]] = addrspacecast ptr addrspace(5) [[ARG]] to ptr
+; CHECK: [[RESULT_VAL:%.*]] = call i32 @vvv_foo(ptr [[ARG_CAST]])
+; CHECK: store i32 [[RESULT_VAL]], ptr [[RESULT]], align 4
+; CHECK: ret void
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_host_entry.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_host_entry.ll
new file mode 100644
index 0000000000000..30a785d5072fa
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_host_entry.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -passes=inputgen-gpu -inputgen-gpu-entry-function=vvv_foo -S 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: inputgen entry kernels are not supported for target 'x86_64-unknown-linux-gnu'
+; CHECK-NOT: __ig_post_load
+; CHECK-NOT: __ig_entry
+; CHECK-LABEL: define i32 @vvv_foo(
+; CHECK: load i32, ptr %a, align 4
+
+define i32 @vvv_foo(ptr noundef %a) {
+entry:
+ %v = load i32, ptr %a, align 4
+ ret i32 %v
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_missing_entry.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_missing_entry.ll
new file mode 100644
index 0000000000000..2182a057819c5
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_missing_entry.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -passes=inputgen-gpu -inputgen-gpu-entry-function=does_not_exist -disable-output 2>&1 | FileCheck %s --check-prefix=MISSING
+; RUN: opt < %s -passes=inputgen-gpu -inputgen-gpu-entry-function=declaration_only -disable-output 2>&1 | FileCheck %s --check-prefix=DECLARATION
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+; MISSING: inputgen entry function 'does_not_exist' was not found or is only a declaration
+; DECLARATION: inputgen entry function 'declaration_only' was not found or is only a declaration
+
+declare i32 @declaration_only(ptr)
+
+define hidden i32 @vvv_foo(ptr noundef %a) {
+entry:
+ %v = load i32, ptr %a, align 4
+ ret i32 %v
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_no_entry.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_no_entry.ll
new file mode 100644
index 0000000000000..522cfc07dd0e8
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_no_entry.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -passes=inputgen-gpu -S | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+; CHECK-NOT: __ig_entry
+; CHECK-LABEL: define hidden i32 @vvv_foo(
+; CHECK: call i64 @__ig_post_load(
+
+define hidden i32 @vvv_foo(ptr noundef %a) {
+entry:
+ %v = load i32, ptr %a, align 4
+ ret i32 %v
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_nvptx_entry.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_nvptx_entry.ll
new file mode 100644
index 0000000000000..1de6d012dd173
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_nvptx_entry.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -passes=inputgen-gpu -inputgen-gpu-entry-function=vvv_foo -S | FileCheck %s
+
+target triple = "nvptx64-nvidia-cuda"
+
+define hidden i32 @vvv_foo(ptr noundef %a) {
+entry:
+ %v = load i32, ptr %a, align 4
+ ret i32 %v
+}
+
+; CHECK-LABEL: define ptx_kernel void @__ig_entry_vvv_foo(
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_runtime_link.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_runtime_link.ll
index 47478dae9bd74..fdf3ed4439d16 100644
--- a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_runtime_link.ll
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_runtime_link.ll
@@ -1,13 +1,19 @@
-; RUN: llvm-as %S/runtimes/inputgen_gpu_callbacks_rt.ll -o %t.rt.bc
-; RUN: opt < %s -passes=inputgen-gpu -inputgen-gpu-runtime-bitcode=%t.rt.bc -S | FileCheck %s
+; RUN: llvm-as %S/runtimes/inputgen_gpu_entry_state_rt.ll -o %t.state.bc
+; RUN: llvm-as %S/runtimes/inputgen_gpu_entry_callbacks_rt.ll -o %t.callbacks.bc
+; RUN: opt < %s -passes=inputgen-gpu -inputgen-gpu-runtime-bitcode=%t.state.bc -inputgen-gpu-runtime-bitcode=%t.callbacks.bc -inputgen-gpu-entry-function=vvv_foo -S | FileCheck %s
-target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
target triple = "amdgcn-amd-amdhsa"
-; CHECK-NOT: @inputgen_
+; CHECK-DAG: @inputgen_buffer = protected addrspace(1) global ptr null
+; CHECK-DAG: @inputgen_buffer_size = protected addrspace(1) global i64 0
+; CHECK-DAG: @inputgen_buffer_offset = protected addrspace(1) global i64 0
+; CHECK-DAG: @inputgen_mode = protected addrspace(1) global i32 0
+; CHECK-DAG: @inputgen_runtime_private = internal global i32 0
; CHECK-LABEL: define hidden i32 @vvv_foo(
; CHECK: call i64 @__ig_post_load(
-; CHECK-LABEL: define protected i64 @__ig_post_load(
+; CHECK-LABEL: define amdgpu_kernel void @__ig_entry_vvv_foo(
+; CHECK: define protected i64 @__ig_post_load(
define hidden i32 @vvv_foo(ptr noundef %a) {
entry:
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_scalar_args.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_scalar_args.ll
new file mode 100644
index 0000000000000..003fd2fa4e7ac
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_scalar_args.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -passes=inputgen-gpu -inputgen-gpu-entry-function=vvv_foo -S | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+define hidden i32 @vvv_foo(i32 %x, i64 %y) {
+entry:
+ %t = trunc i64 %y to i32
+ %r = add i32 %x, %t
+ ret i32 %r
+}
+
+; CHECK-LABEL: define amdgpu_kernel void @__ig_entry_vvv_foo(
+; CHECK: [[RESULT_VAL:%.*]] = call i32 @vvv_foo(i32 0, i64 0)
+; CHECK: store i32 [[RESULT_VAL]], ptr [[RESULT:%.*]], align 4
diff --git a/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_void_return.ll b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_void_return.ll
new file mode 100644
index 0000000000000..3b655a1cf9c30
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/inputgen_gpu_void_return.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -passes=inputgen-gpu -inputgen-gpu-entry-function=vvv_foo -S | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+define hidden void @vvv_foo(ptr noundef %a) {
+entry:
+ store i32 1, ptr %a, align 4
+ ret void
+}
+
+; CHECK-LABEL: define amdgpu_kernel void @__ig_entry_vvv_foo(
+; CHECK: call void @vvv_foo(
+; CHECK-NEXT: ret void
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_entry_callbacks_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_entry_callbacks_rt.ll
new file mode 100644
index 0000000000000..413c5460f9d08
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_entry_callbacks_rt.ll
@@ -0,0 +1,13 @@
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+ at inputgen_runtime_private = internal global i32 0
+
+define protected i64 @__ig_post_load(i64 %value, i64 %value_size,
+ i32 %value_type_id, i32 %id) {
+entry:
+ %private = load i32, ptr @inputgen_runtime_private, align 4
+ %next = add i32 %private, 1
+ store i32 %next, ptr @inputgen_runtime_private, align 4
+ ret i64 %value
+}
diff --git a/llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_entry_state_rt.ll b/llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_entry_state_rt.ll
new file mode 100644
index 0000000000000..bbf56695a95bb
--- /dev/null
+++ b/llvm/test/Instrumentation/Instrumentor/runtimes/inputgen_gpu_entry_state_rt.ll
@@ -0,0 +1,7 @@
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+ at inputgen_buffer = protected addrspace(1) global ptr null
+ at inputgen_buffer_size = protected addrspace(1) global i64 0
+ at inputgen_buffer_offset = protected addrspace(1) global i64 0
+ at inputgen_mode = protected addrspace(1) global i32 0
More information about the llvm-commits
mailing list