[clang] [llvm] [SPIRV] Handle AMDGPU buffers in AMDGCN flavoured SPIR-V (PR #199376)
Alex Voicu via cfe-commits
cfe-commits at lists.llvm.org
Sat May 23 14:33:06 PDT 2026
https://github.com/AlexVlx updated https://github.com/llvm/llvm-project/pull/199376
>From 3e0abe10c53e08cc1ebf16662d8d58cd7245f70f Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Thu, 15 May 2025 23:27:42 +0100
Subject: [PATCH 01/29] Add pass which forwards unimplemented math builtins /
libcalls to the HIPSTDPAR runtime component.
---
.../llvm/Transforms/HipStdPar/HipStdPar.h | 7 +
llvm/lib/Passes/PassRegistry.def | 1 +
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 8 +-
llvm/lib/Transforms/HipStdPar/HipStdPar.cpp | 117 +++++++++
llvm/test/Transforms/HipStdPar/math-fixup.ll | 240 ++++++++++++++++++
5 files changed, 371 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/HipStdPar/math-fixup.ll
diff --git a/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h b/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h
index 5ff38bdf04812..27195051ed7eb 100644
--- a/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h
+++ b/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h
@@ -40,6 +40,13 @@ class HipStdParAllocationInterpositionPass
static bool isRequired() { return true; }
};
+class HipStdParMathFixupPass : public PassInfoMixin<HipStdParMathFixupPass> {
+public:
+ PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+
+ static bool isRequired() { return true; }
+};
+
} // namespace llvm
#endif // LLVM_TRANSFORMS_HIPSTDPAR_HIPSTDPAR_H
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 94dabe290213d..3acdbf4d49fde 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -80,6 +80,7 @@ MODULE_PASS("global-merge-func", GlobalMergeFuncPass())
MODULE_PASS("globalopt", GlobalOptPass())
MODULE_PASS("globalsplit", GlobalSplitPass())
MODULE_PASS("hipstdpar-interpose-alloc", HipStdParAllocationInterpositionPass())
+MODULE_PASS("hipstdpar-math-fixup", HipStdParMathFixupPass())
MODULE_PASS("hipstdpar-select-accelerator-code",
HipStdParAcceleratorCodeSelectionPass())
MODULE_PASS("hotcoldsplit", HotColdSplittingPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index ccb251b730f16..c3f8cee1e1783 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -819,8 +819,10 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
// When we are not using -fgpu-rdc, we can run accelerator code
// selection relatively early, but still after linking to prevent
// eager removal of potentially reachable symbols.
- if (EnableHipStdPar)
+ if (EnableHipStdPar) {
+ PM.addPass(HipStdParMathFixupPass());
PM.addPass(HipStdParAcceleratorCodeSelectionPass());
+ }
PM.addPass(AMDGPUPrintfRuntimeBindingPass());
}
@@ -899,8 +901,10 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
// selection after linking to prevent, otherwise we end up removing
// potentially reachable symbols that were exported as external in other
// modules.
- if (EnableHipStdPar)
+ if (EnableHipStdPar) {
+ PM.addPass(HipStdParMathFixupPass());
PM.addPass(HipStdParAcceleratorCodeSelectionPass());
+ }
// We want to support the -lto-partitions=N option as "best effort".
// For that, we need to lower LDS earlier in the pipeline before the
// module is partitioned for codegen.
diff --git a/llvm/lib/Transforms/HipStdPar/HipStdPar.cpp b/llvm/lib/Transforms/HipStdPar/HipStdPar.cpp
index 5a87cf8c83d79..815878089c69e 100644
--- a/llvm/lib/Transforms/HipStdPar/HipStdPar.cpp
+++ b/llvm/lib/Transforms/HipStdPar/HipStdPar.cpp
@@ -37,6 +37,16 @@
// memory that ends up in one of the runtime equivalents, since this can
// happen if e.g. a library that was compiled without interposition returns
// an allocation that can be validly passed to `free`.
+//
+// 3. MathFixup (required): Some accelerators might have an incomplete
+// implementation for the intrinsics used to implement some of the math
+// functions in <cmath> / their corresponding libcall lowerings. Since this
+// can vary quite significantly between accelerators, we replace calls to a
+// set of intrinsics / lib functions known to be problematic with calls to a
+// HIPSTDPAR specific forwarding layer, which gives an uniform interface for
+// accelerators to implement in their own runtime components. This pass
+// should run before AcceleratorCodeSelection so as to prevent the spurious
+// removal of the HIPSTDPAR specific forwarding functions.
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/HipStdPar/HipStdPar.h"
@@ -48,6 +58,7 @@
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
+#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
@@ -321,3 +332,109 @@ HipStdParAllocationInterpositionPass::run(Module &M, ModuleAnalysisManager&) {
return PreservedAnalyses::none();
}
+
+static constexpr std::pair<StringLiteral, StringLiteral> MathLibToHipStdPar[]{
+ {"acosh", "__hipstdpar_acosh_f64"},
+ {"acoshf", "__hipstdpar_acosh_f32"},
+ {"asinh", "__hipstdpar_asinh_f64"},
+ {"asinhf", "__hipstdpar_asinh_f32"},
+ {"atanh", "__hipstdpar_atanh_f64"},
+ {"atanhf", "__hipstdpar_atanh_f32"},
+ {"cbrt", "__hipstdpar_cbrt_f64"},
+ {"cbrtf", "__hipstdpar_cbrt_f32"},
+ {"erf", "__hipstdpar_erf_f64"},
+ {"erff", "__hipstdpar_erf_f32"},
+ {"erfc", "__hipstdpar_erfc_f64"},
+ {"erfcf", "__hipstdpar_erfc_f32"},
+ {"fdim", "__hipstdpar_fdim_f64"},
+ {"fdimf", "__hipstdpar_fdim_f32"},
+ {"expm1", "__hipstdpar_expm1_f64"},
+ {"expm1f", "__hipstdpar_expm1_f32"},
+ {"hypot", "__hipstdpar_hypot_f64"},
+ {"hypotf", "__hipstdpar_hypot_f32"},
+ {"ilogb", "__hipstdpar_ilogb_f64"},
+ {"ilogbf", "__hipstdpar_ilogb_f32"},
+ {"lgamma", "__hipstdpar_lgamma_f64"},
+ {"lgammaf", "__hipstdpar_lgamma_f32"},
+ {"log1p", "__hipstdpar_log1p_f64"},
+ {"log1pf", "__hipstdpar_log1p_f32"},
+ {"logb", "__hipstdpar_logb_f64"},
+ {"logbf", "__hipstdpar_logb_f32"},
+ {"nextafter", "__hipstdpar_nextafter_f64"},
+ {"nextafterf", "__hipstdpar_nextafter_f32"},
+ {"nexttoward", "__hipstdpar_nexttoward_f64"},
+ {"nexttowardf", "__hipstdpar_nexttoward_f32"},
+ {"remainder", "__hipstdpar_remainder_f64"},
+ {"remainderf", "__hipstdpar_remainder_f32"},
+ {"remquo", "__hipstdpar_remquo_f64"},
+ {"remquof", "__hipstdpar_remquo_f32"},
+ {"scalbln", "__hipstdpar_scalbln_f64"},
+ {"scalblnf", "__hipstdpar_scalbln_f32"},
+ {"scalbn", "__hipstdpar_scalbn_f64"},
+ {"scalbnf", "__hipstdpar_scalbn_f32"},
+ {"tgamma", "__hipstdpar_tgamma_f64"},
+ {"tgammaf", "__hipstdpar_tgamma_f32"}};
+
+PreservedAnalyses HipStdParMathFixupPass::run(Module &M,
+ ModuleAnalysisManager &) {
+ if (M.empty())
+ return PreservedAnalyses::all();
+
+ SmallVector<std::pair<Function *, std::string>> ToReplace;
+ for (auto &&F : M) {
+ if (!F.hasName())
+ continue;
+
+ auto N = F.getName().str();
+ auto ID = F.getIntrinsicID();
+
+ switch (ID) {
+ case Intrinsic::not_intrinsic: {
+ auto It = find_if(MathLibToHipStdPar,
+ [&](auto &&M) { return M.first == N; });
+ if (It == std::cend(MathLibToHipStdPar))
+ continue;
+ ToReplace.emplace_back(&F, It->second);
+ break;
+ }
+ case Intrinsic::acos:
+ case Intrinsic::asin:
+ case Intrinsic::atan:
+ case Intrinsic::atan2:
+ case Intrinsic::cosh:
+ case Intrinsic::modf:
+ case Intrinsic::sinh:
+ case Intrinsic::tan:
+ case Intrinsic::tanh:
+ break;
+ default: {
+ if (F.getReturnType()->isDoubleTy()) {
+ switch (ID) {
+ case Intrinsic::cos:
+ case Intrinsic::exp:
+ case Intrinsic::exp2:
+ case Intrinsic::log:
+ case Intrinsic::log10:
+ case Intrinsic::log2:
+ case Intrinsic::pow:
+ case Intrinsic::sin:
+ break;
+ default:
+ continue;
+ }
+ break;
+ }
+ continue;
+ }
+ }
+
+ llvm::replace(N, '.', '_');
+ N.replace(0, sizeof("llvm"), "__hipstdpar_");
+ ToReplace.emplace_back(&F, std::move(N));
+ }
+ for (auto &&F : ToReplace)
+ F.first->replaceAllUsesWith(M.getOrInsertFunction(
+ F.second, F.first->getFunctionType()).getCallee());
+
+ return PreservedAnalyses::none();
+}
\ No newline at end of file
diff --git a/llvm/test/Transforms/HipStdPar/math-fixup.ll b/llvm/test/Transforms/HipStdPar/math-fixup.ll
new file mode 100644
index 0000000000000..e0e2ca79c0843
--- /dev/null
+++ b/llvm/test/Transforms/HipStdPar/math-fixup.ll
@@ -0,0 +1,240 @@
+; RUN: opt -S -passes=hipstdpar-math-fixup %s | FileCheck %s
+
+define void @foo(double noundef %dbl, float noundef %flt, i32 noundef %quo) #0 {
+; CHECK-LABEL: define void @foo(
+; CHECK-SAME: double noundef [[DBL:%.*]], float noundef [[FLT:%.*]], i32 noundef [[QUO:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[QUO_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT: store i32 [[QUO]], ptr [[QUO_ADDR]], align 4
+entry:
+ %quo.addr = alloca i32, align 4
+ store i32 %quo, ptr %quo.addr, align 4
+ ; CHECK-NEXT: [[TMP0:%.*]] = tail call contract double @llvm.fabs.f64(double [[DBL]])
+ %0 = tail call contract double @llvm.fabs.f64(double %dbl)
+ ; CHECK-NEXT: [[TMP1:%.*]] = tail call contract float @llvm.fabs.f32(float [[FLT]])
+ %1 = tail call contract float @llvm.fabs.f32(float %flt)
+ ; CHECK-NEXT: [[CALL:%.*]] = tail call contract double @__hipstdpar_remainder_f64(double noundef [[TMP0]], double noundef [[TMP0]]) #[[ATTR4:[0-9]+]]
+ %call = tail call contract double @remainder(double noundef %0, double noundef %0) #4
+ ; CHECK-NEXT: [[CALL1:%.*]] = tail call contract float @__hipstdpar_remainder_f32(float noundef [[TMP1]], float noundef [[TMP1]]) #[[ATTR4]]
+ %call1 = tail call contract float @remainderf(float noundef %1, float noundef %1) #4
+ ; CHECK-NEXT: [[CALL2:%.*]] = call contract double @__hipstdpar_remquo_f64(double noundef [[CALL]], double noundef [[CALL]], ptr noundef nonnull [[QUO_ADDR]]) #[[ATTR3:[0-9]+]]
+ %call2 = call contract double @remquo(double noundef %call, double noundef %call, ptr noundef nonnull %quo.addr) #5
+ ; CHECK-NEXT: [[CALL3:%.*]] = call contract float @__hipstdpar_remquo_f32(float noundef [[CALL1]], float noundef [[CALL1]], ptr noundef nonnull [[QUO_ADDR]]) #[[ATTR3]]
+ %call3 = call contract float @remquof(float noundef %call1, float noundef %call1, ptr noundef nonnull %quo.addr) #5
+ ; CHECK-NEXT: [[TMP2:%.*]] = call contract double @llvm.fma.f64(double [[CALL2]], double [[CALL2]], double [[CALL2]])
+ %2 = call contract double @llvm.fma.f64(double %call2, double %call2, double %call2)
+ ; CHECK-NEXT: [[TMP3:%.*]] = call contract float @llvm.fma.f32(float [[CALL3]], float [[CALL3]], float [[CALL3]])
+ %3 = call contract float @llvm.fma.f32(float %call3, float %call3, float %call3)
+ ; CHECK-NEXT: [[CALL4:%.*]] = call contract double @__hipstdpar_fdim_f64(double noundef [[TMP2]], double noundef [[TMP2]]) #[[ATTR4]]
+ %call4 = call contract double @fdim(double noundef %2, double noundef %2) #4
+ ; CHECK-NEXT: [[CALL5:%.*]] = call contract float @__hipstdpar_fdim_f32(float noundef [[TMP3]], float noundef [[TMP3]]) #[[ATTR4]]
+ %call5 = call contract float @fdimf(float noundef %3, float noundef %3) #4
+ ; CHECK-NEXT: [[TMP4:%.*]] = call contract double @__hipstdpar_exp_f64(double [[CALL4]])
+ %4 = call contract double @llvm.exp.f64(double %call4)
+ ; CHECK-NEXT: [[TMP5:%.*]] = call contract float @llvm.exp.f32(float [[CALL5]])
+ %5 = call contract float @llvm.exp.f32(float %call5)
+ ; CHECK-NEXT: [[TMP6:%.*]] = call contract double @__hipstdpar_exp2_f64(double [[TMP4]])
+ %6 = call contract double @llvm.exp2.f64(double %4)
+ ; CHECK-NEXT: [[TMP7:%.*]] = call contract float @llvm.exp2.f32(float [[TMP5]])
+ %7 = call contract float @llvm.exp2.f32(float %5)
+ ; CHECK-NEXT: [[CALL6:%.*]] = call contract double @__hipstdpar_expm1_f64(double noundef [[TMP6]]) #[[ATTR4]]
+ %call6 = call contract double @expm1(double noundef %6) #4
+ ; CHECK-NEXT: [[CALL7:%.*]] = call contract float @__hipstdpar_expm1_f32(float noundef [[TMP7]]) #[[ATTR4]]
+ %call7 = call contract float @expm1f(float noundef %7) #4
+ ; CHECK-NEXT: [[TMP8:%.*]] = call contract double @__hipstdpar_log_f64(double [[CALL6]])
+ %8 = call contract double @llvm.log.f64(double %call6)
+ ; CHECK-NEXT: [[TMP9:%.*]] = call contract float @llvm.log.f32(float [[CALL7]])
+ %9 = call contract float @llvm.log.f32(float %call7)
+ ; CHECK-NEXT: [[TMP10:%.*]] = call contract double @__hipstdpar_log10_f64(double [[TMP8]])
+ %10 = call contract double @llvm.log10.f64(double %8)
+ ; CHECK-NEXT: [[TMP11:%.*]] = call contract float @llvm.log10.f32(float [[TMP9]])
+ %11 = call contract float @llvm.log10.f32(float %9)
+ ; CHECK-NEXT: [[TMP12:%.*]] = call contract double @__hipstdpar_log2_f64(double [[TMP10]])
+ %12 = call contract double @llvm.log2.f64(double %10)
+ ; CHECK-NEXT: [[TMP13:%.*]] = call contract float @llvm.log2.f32(float [[TMP11]])
+ %13 = call contract float @llvm.log2.f32(float %11)
+ ; CHECK-NEXT: [[CALL8:%.*]] = call contract double @__hipstdpar_log1p_f64(double noundef [[TMP12]]) #[[ATTR4]]
+ %call8 = call contract double @log1p(double noundef %12) #4
+ ; CHECK-NEXT: [[CALL9:%.*]] = call contract float @__hipstdpar_log1p_f32(float noundef [[TMP13]]) #[[ATTR4]]
+ %call9 = call contract float @log1pf(float noundef %13) #4
+ ; CHECK-NEXT: [[TMP14:%.*]] = call contract float @llvm.pow.f32(float [[CALL9]], float [[CALL9]])
+ %14 = call contract float @llvm.pow.f32(float %call9, float %call9)
+ ; CHECK-NEXT: [[TMP15:%.*]] = call contract double @llvm.sqrt.f64(double [[CALL8]])
+ %15 = call contract double @llvm.sqrt.f64(double %call8)
+ ; CHECK-NEXT: [[TMP16:%.*]] = call contract float @llvm.sqrt.f32(float [[TMP14]])
+ %16 = call contract float @llvm.sqrt.f32(float %14)
+ ; CHECK-NEXT: [[CALL10:%.*]] = call contract double @__hipstdpar_cbrt_f64(double noundef [[TMP15]]) #[[ATTR4]]
+ %call10 = call contract double @cbrt(double noundef %15) #4
+ ; CHECK-NEXT: [[CALL11:%.*]] = call contract float @__hipstdpar_cbrt_f32(float noundef [[TMP16]]) #[[ATTR4]]
+ %call11 = call contract float @cbrtf(float noundef %16) #4
+ ; CHECK-NEXT: [[CALL12:%.*]] = call contract double @__hipstdpar_hypot_f64(double noundef [[CALL10]], double noundef [[CALL10]]) #[[ATTR4]]
+ %call12 = call contract double @hypot(double noundef %call10, double noundef %call10) #4
+ ; CHECK-NEXT: [[CALL13:%.*]] = call contract float @__hipstdpar_hypot_f32(float noundef [[CALL11]], float noundef [[CALL11]]) #[[ATTR4]]
+ %call13 = call contract float @hypotf(float noundef %call11, float noundef %call11) #4
+ ; CHECK-NEXT: [[TMP17:%.*]] = call contract float @llvm.sin.f32(float [[CALL13]])
+ %17 = call contract float @llvm.sin.f32(float %call13)
+ ; CHECK-NEXT: [[TMP18:%.*]] = call contract float @llvm.cos.f32(float [[TMP17]])
+ %18 = call contract float @llvm.cos.f32(float %17)
+ ; CHECK-NEXT: [[TMP19:%.*]] = call contract double @__hipstdpar_tan_f64(double [[CALL12]])
+ %19 = call contract double @llvm.tan.f64(double %call12)
+ ; CHECK-NEXT: [[TMP20:%.*]] = call contract double @__hipstdpar_asin_f64(double [[TMP19]])
+ %20 = call contract double @llvm.asin.f64(double %19)
+ ; CHECK-NEXT: [[TMP21:%.*]] = call contract double @__hipstdpar_acos_f64(double [[TMP20]])
+ %21 = call contract double @llvm.acos.f64(double %20)
+ ; CHECK-NEXT: [[TMP22:%.*]] = call contract double @__hipstdpar_atan_f64(double [[TMP21]])
+ %22 = call contract double @llvm.atan.f64(double %21)
+ ; CHECK-NEXT: [[TMP23:%.*]] = call contract double @__hipstdpar_atan2_f64(double [[TMP22]], double [[TMP22]])
+ %23 = call contract double @llvm.atan2.f64(double %22, double %22)
+ ; CHECK-NEXT: [[TMP24:%.*]] = call contract double @__hipstdpar_sinh_f64(double [[TMP23]])
+ %24 = call contract double @llvm.sinh.f64(double %23)
+ ; CHECK-NEXT: [[TMP25:%.*]] = call contract double @__hipstdpar_cosh_f64(double [[TMP24]])
+ %25 = call contract double @llvm.cosh.f64(double %24)
+ ; CHECK-NEXT: [[TMP26:%.*]] = call contract double @__hipstdpar_tanh_f64(double [[TMP25]])
+ %26 = call contract double @llvm.tanh.f64(double %25)
+ ; CHECK-NEXT: [[CALL14:%.*]] = call contract double @__hipstdpar_asinh_f64(double noundef [[TMP26]]) #[[ATTR4]]
+ %call14 = call contract double @asinh(double noundef %26) #4
+ ; CHECK-NEXT: [[CALL15:%.*]] = call contract float @__hipstdpar_asinh_f32(float noundef [[TMP18]]) #[[ATTR4]]
+ %call15 = call contract float @asinhf(float noundef %18) #4
+ ; CHECK-NEXT: [[CALL16:%.*]] = call contract double @__hipstdpar_acosh_f64(double noundef [[CALL14]]) #[[ATTR4]]
+ %call16 = call contract double @acosh(double noundef %call14) #4
+ ; CHECK-NEXT: [[CALL17:%.*]] = call contract float @__hipstdpar_acosh_f32(float noundef [[CALL15]]) #[[ATTR4]]
+ %call17 = call contract float @acoshf(float noundef %call15) #4
+ ; CHECK-NEXT: [[CALL18:%.*]] = call contract double @__hipstdpar_atanh_f64(double noundef [[CALL16]]) #[[ATTR4]]
+ %call18 = call contract double @atanh(double noundef %call16) #4
+ ; CHECK-NEXT: [[CALL19:%.*]] = call contract float @__hipstdpar_atanh_f32(float noundef [[CALL17]]) #[[ATTR4]]
+ %call19 = call contract float @atanhf(float noundef %call17) #4
+ ; CHECK-NEXT: [[CALL20:%.*]] = call contract double @__hipstdpar_erf_f64(double noundef [[CALL18]]) #[[ATTR4]]
+ %call20 = call contract double @erf(double noundef %call18) #4
+ ; CHECK-NEXT: [[CALL21:%.*]] = call contract float @__hipstdpar_erf_f32(float noundef [[CALL19]]) #[[ATTR4]]
+ %call21 = call contract float @erff(float noundef %call19) #4
+ ; CHECK-NEXT: [[CALL22:%.*]] = call contract double @__hipstdpar_erfc_f64(double noundef [[CALL20]]) #[[ATTR4]]
+ %call22 = call contract double @erfc(double noundef %call20) #4
+ ; CHECK-NEXT: [[CALL23:%.*]] = call contract float @__hipstdpar_erfc_f32(float noundef [[CALL21]]) #[[ATTR4]]
+ %call23 = call contract float @erfcf(float noundef %call21) #4
+ ; CHECK-NEXT: [[CALL24:%.*]] = call contract double @__hipstdpar_tgamma_f64(double noundef [[CALL22]]) #[[ATTR4]]
+ %call24 = call contract double @tgamma(double noundef %call22) #4
+ ; CHECK-NEXT: [[CALL25:%.*]] = call contract float @__hipstdpar_tgamma_f32(float noundef [[CALL23]]) #[[ATTR4]]
+ %call25 = call contract float @tgammaf(float noundef %call23) #4
+ ; CHECK-NEXT: [[CALL26:%.*]] = call contract double @__hipstdpar_lgamma_f64(double noundef [[CALL24]]) #[[ATTR3]]
+ %call26 = call contract double @lgamma(double noundef %call24) #5
+ ; CHECK-NEXT: [[CALL27:%.*]] = call contract float @__hipstdpar_lgamma_f32(float noundef [[CALL25]]) #[[ATTR3]]
+ %call27 = call contract float @lgammaf(float noundef %call25) #5
+ ret void
+}
+
+declare double @llvm.fabs.f64(double) #1
+
+declare float @llvm.fabs.f32(float) #1
+
+declare hidden double @remainder(double noundef, double noundef) local_unnamed_addr #2
+
+declare hidden float @remainderf(float noundef, float noundef) local_unnamed_addr #2
+
+declare hidden double @remquo(double noundef, double noundef, ptr noundef) local_unnamed_addr #3
+
+declare hidden float @remquof(float noundef, float noundef, ptr noundef) local_unnamed_addr #3
+
+declare double @llvm.fma.f64(double, double, double) #1
+
+declare float @llvm.fma.f32(float, float, float) #1
+
+declare hidden double @fdim(double noundef, double noundef) local_unnamed_addr #2
+
+declare hidden float @fdimf(float noundef, float noundef) local_unnamed_addr #2
+
+declare double @llvm.exp.f64(double) #1
+
+declare float @llvm.exp.f32(float) #1
+
+declare double @llvm.exp2.f64(double) #1
+
+declare float @llvm.exp2.f32(float) #1
+
+declare hidden double @expm1(double noundef) local_unnamed_addr #2
+
+declare hidden float @expm1f(float noundef) local_unnamed_addr #2
+
+declare double @llvm.log.f64(double) #1
+
+declare float @llvm.log.f32(float) #1
+
+declare double @llvm.log10.f64(double) #1
+
+declare float @llvm.log10.f32(float) #1
+
+declare double @llvm.log2.f64(double) #1
+
+declare float @llvm.log2.f32(float) #1
+
+declare hidden double @log1p(double noundef) local_unnamed_addr #2
+
+declare hidden float @log1pf(float noundef) local_unnamed_addr #2
+
+declare float @llvm.pow.f32(float, float) #1
+
+declare double @llvm.sqrt.f64(double) #1
+
+declare float @llvm.sqrt.f32(float) #1
+
+declare hidden double @cbrt(double noundef) local_unnamed_addr #2
+
+declare hidden float @cbrtf(float noundef) local_unnamed_addr #2
+
+declare hidden double @hypot(double noundef, double noundef) local_unnamed_addr #2
+
+declare hidden float @hypotf(float noundef, float noundef) local_unnamed_addr #2
+
+declare float @llvm.sin.f32(float) #1
+
+declare float @llvm.cos.f32(float) #1
+
+declare double @llvm.tan.f64(double) #1
+
+declare double @llvm.asin.f64(double) #1
+
+declare double @llvm.acos.f64(double) #1
+
+declare double @llvm.atan.f64(double) #1
+
+declare double @llvm.atan2.f64(double, double) #1
+
+declare double @llvm.sinh.f64(double) #1
+
+declare double @llvm.cosh.f64(double) #1
+
+declare double @llvm.tanh.f64(double) #1
+
+declare hidden double @asinh(double noundef) local_unnamed_addr #2
+
+declare hidden float @asinhf(float noundef) local_unnamed_addr #2
+
+declare hidden double @acosh(double noundef) local_unnamed_addr #2
+
+declare hidden float @acoshf(float noundef) local_unnamed_addr #2
+
+declare hidden double @atanh(double noundef) local_unnamed_addr #2
+
+declare hidden float @atanhf(float noundef) local_unnamed_addr #2
+
+declare hidden double @erf(double noundef) local_unnamed_addr #2
+
+declare hidden float @erff(float noundef) local_unnamed_addr #2
+
+declare hidden double @erfc(double noundef) local_unnamed_addr #2
+
+declare hidden float @erfcf(float noundef) local_unnamed_addr #2
+
+declare hidden double @tgamma(double noundef) local_unnamed_addr #2
+
+declare hidden float @tgammaf(float noundef) local_unnamed_addr #2
+
+declare hidden double @lgamma(double noundef) local_unnamed_addr #3
+
+declare hidden float @lgammaf(float noundef) local_unnamed_addr #3
+
+attributes #0 = { convergent mustprogress norecurse nounwind }
+attributes #1 = { mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #2 = { convergent mustprogress nofree nounwind willreturn memory(none) }
+attributes #3 = { convergent nounwind }
+attributes #4 = { convergent nounwind willreturn memory(none) }
+attributes #5 = { convergent nounwind }
>From f951184fb039c24a9b25b59399d5c1a28cef13a7 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Wed, 17 Dec 2025 01:34:41 +0000
Subject: [PATCH 02/29] Handle `COPY` when selecting `UtoPtr` as the
`SpecConstantOp` op.
---
llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 7 ++++---
.../CodeGen/SPIRV/transcoding/ConvertPtrInGlobalInit.ll | 3 ++-
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 2e4563795e8f0..aae025602b763 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1099,6 +1099,7 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
UseEnd = MRI->use_instr_end();
UseIt != UseEnd; UseIt = std::next(UseIt)) {
if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
+ (*UseIt).getOpcode() == SPIRV::OpSpecConstantOp ||
(*UseIt).getOpcode() == SPIRV::OpVariable) {
IsGVInit = true;
break;
@@ -1407,9 +1408,9 @@ bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
MRI->def_instr_begin(SrcReg);
DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) {
unsigned DefOpCode = DefIt->getOpcode();
- if (DefOpCode == SPIRV::ASSIGN_TYPE) {
- // We need special handling to look through the type assignment and see
- // if this is a constant or a global
+ if (DefOpCode == SPIRV::ASSIGN_TYPE || DefOpCode == TargetOpcode::COPY) {
+ // We need special handling to look through the type assignment or the
+ // COPY pseudo-op and see if this is a constant or a global
if (auto *VRD = getVRegDef(*MRI, DefIt->getOperand(1).getReg()))
DefOpCode = VRD->getOpcode();
}
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/ConvertPtrInGlobalInit.ll b/llvm/test/CodeGen/SPIRV/transcoding/ConvertPtrInGlobalInit.ll
index f397030c7bdb1..23aaa5573a83c 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/ConvertPtrInGlobalInit.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/ConvertPtrInGlobalInit.ll
@@ -12,11 +12,11 @@
; CHECK: %[[VtblTy:[0-9]+]] = OpTypeStruct %[[ArrTy]] %[[ArrTy]] %[[ArrTy]] %[[ArrTy]] %[[ArrTy]]
; CHECK: %[[Int64Ty:[0-9]+]] = OpTypeInt 64 0
; CHECK: %[[GlobVtblPtrTy:[0-9]+]] = OpTypePointer CrossWorkgroup %[[VtblTy]]
+; CHECK: %[[Const184:[0-9]+]] = OpConstant %[[Int64Ty]] 184
; CHECK: %[[ConstMinus184:[0-9]+]] = OpConstant %[[Int64Ty]] 18446744073709551432
; CHECK: %[[ConstMinus16:[0-9]+]] = OpConstant %[[Int64Ty]] 18446744073709551600
; CHECK: %[[Const168:[0-9]+]] = OpConstant %[[Int64Ty]] 168
; CHECK: %[[Nullptr:[0-9]+]] = OpConstantNull %[[GlobInt8PtrTy]]
-; CHECK: %[[Const184:[0-9]+]] = OpConstant %[[Int64Ty]] 184
; CHECK: %[[Const184toPtr:[0-9]+]] = OpSpecConstantOp %[[GlobInt8PtrTy]] ConvertUToPtr %[[Const184]]
; CHECK: %[[Const168toPtr:[0-9]+]] = OpSpecConstantOp %[[GlobInt8PtrTy]] ConvertUToPtr %[[Const168]]
; CHECK: %[[ConstMinus16toPtr:[0-9]+]] = OpSpecConstantOp %[[GlobInt8PtrTy]] ConvertUToPtr %[[ConstMinus16]]
@@ -33,6 +33,7 @@
[5 x ptr addrspace(1)] [ptr addrspace(1) inttoptr (i64 184 to ptr addrspace(1)), ptr addrspace(1) null, ptr addrspace(1) null, ptr addrspace(1) null, ptr addrspace(1) null],
[5 x ptr addrspace(1)] [ptr addrspace(1) inttoptr (i64 168 to ptr addrspace(1)), ptr addrspace(1) inttoptr (i64 -16 to ptr addrspace(1)), ptr addrspace(1) null, ptr addrspace(1) null, ptr addrspace(1) null],
[5 x ptr addrspace(1)] [ptr addrspace(1) inttoptr (i64 -184 to ptr addrspace(1)), ptr addrspace(1) inttoptr (i64 -184 to ptr addrspace(1)), ptr addrspace(1) null, ptr addrspace(1) null, ptr addrspace(1) null] }
+ at VTT = linkonce_odr unnamed_addr addrspace(1) constant [1 x ptr addrspace(1)][ptr addrspace(1) getelementptr inbounds inrange(-24, 16) ({ [5 x ptr addrspace(1)], [5 x ptr addrspace(1)], [5 x ptr addrspace(1)], [5 x ptr addrspace(1)], [5 x ptr addrspace(1)] }, ptr addrspace(1) @vtable, i32 0, i32 4, i32 3)]
define linkonce_odr spir_func void @foo(ptr addrspace(4) %this) {
entry:
>From 86cf09343456da3273e52d89592dbe23e792bc32 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Thu, 8 Jan 2026 14:33:07 +0000
Subject: [PATCH 03/29] Update
llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
Co-authored-by: Marcos Maronas <marcos.maronas at intel.com>
---
llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 964c555883b85..100057f6d1a39 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1412,7 +1412,7 @@ bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
unsigned DefOpCode = DefIt->getOpcode();
if (DefOpCode == SPIRV::ASSIGN_TYPE || DefOpCode == TargetOpcode::COPY) {
// We need special handling to look through the type assignment or the
- // COPY pseudo-op and see if this is a constant or a global
+ // COPY pseudo-op and see if this is a constant or a global.
if (auto *VRD = getVRegDef(*MRI, DefIt->getOperand(1).getReg()))
DefOpCode = VRD->getOpcode();
}
>From 87a88c667d44c0996a391184b2930451d43ded69 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Sat, 10 Jan 2026 02:48:27 +0000
Subject: [PATCH 04/29] Sneak `externally_initialized` via `HostAccessINTEL`.
---
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 10 ++++++++
.../CodeGen/SPIRV/externally-initialized.ll | 25 +++++++++++++++++++
2 files changed, 35 insertions(+)
create mode 100644 llvm/test/CodeGen/SPIRV/externally-initialized.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 100057f6d1a39..265e9b49ae52d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -4791,6 +4791,16 @@ bool SPIRVInstructionSelector::selectGlobalValue(
Register Reg = GR.buildGlobalVariable(
ResVReg, ResType, GlobalIdent, GV, StorageClass, Init,
GlobalVar->isConstant(), LnkType, MIRBuilder, true);
+ // TODO: For AMDGCN, we pipe externally_initialized through via
+ // HostAccessINTEL, with ReadWrite (3) access, which is we then handle during
+ // reverse translation. We should remove this once SPIR-V gains the ability to
+ // express the concept.
+ if (GlobalVar->isExternallyInitialized() &&
+ STI.getTargetTriple().getVendor() == Triple::AMD) {
+ buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::HostAccessINTEL, {3u});
+ MachineInstrBuilder MIB(*MF, --MIRBuilder.getInsertPt());
+ addStringImm(GV->getName(), MIB);
+ }
return Reg.isValid();
}
diff --git a/llvm/test/CodeGen/SPIRV/externally-initialized.ll b/llvm/test/CodeGen/SPIRV/externally-initialized.ll
new file mode 100644
index 0000000000000..ac547417359d3
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/externally-initialized.ll
@@ -0,0 +1,25 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s --check-prefix=CHECK-AMDGCNSPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-SPIRV: OpName %[[#G:]] "G"
+; CHECK-SPIRV-NOT: OpDecorate %[[#G]] ReferencedIndirectlyINTEL
+; CHECK-SPIRV-DAG: %[[#G]] = OpVariable
+
+; CHECK-AMDGCNSPIRV: OpExtension "SPV_INTEL_global_variable_host_access"
+; CHECK-AMDGCNSPIRV: OpName %[[#G:]] "G"
+; CHECK-AMDGCNSPIRV: OpDecorate %[[#G]] HostAccessINTEL 3 "G"
+; CHECK-AMDGCNSPIRV-DAG: %[[#G]] = OpVariable
+
+
+ at G = external addrspace(1) externally_initialized global i32
+
+define spir_func i32 @foo() {
+ %r = load i32, ptr addrspace(1) @G
+ ret i32 %r
+}
>From 004c880ac2a51fc9628a0ac1f3801f2451c48dbe Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Mon, 12 Jan 2026 15:14:47 +0200
Subject: [PATCH 05/29] Apply review suggestion.
---
llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 265e9b49ae52d..84813905df1fe 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -4797,7 +4797,8 @@ bool SPIRVInstructionSelector::selectGlobalValue(
// express the concept.
if (GlobalVar->isExternallyInitialized() &&
STI.getTargetTriple().getVendor() == Triple::AMD) {
- buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::HostAccessINTEL, {3u});
+ constexpr unsigned ReadWriteINTEL = 3u;
+ buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::HostAccessINTEL, {ReadWriteINTEL});
MachineInstrBuilder MIB(*MF, --MIRBuilder.getInsertPt());
addStringImm(GV->getName(), MIB);
}
>From 8b1cdf0690a58aa405167126eea4e1f0a01538e6 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Mon, 12 Jan 2026 18:08:38 +0200
Subject: [PATCH 06/29] Fix formatting.
---
llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 84813905df1fe..98b5bfd678135 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -4798,7 +4798,8 @@ bool SPIRVInstructionSelector::selectGlobalValue(
if (GlobalVar->isExternallyInitialized() &&
STI.getTargetTriple().getVendor() == Triple::AMD) {
constexpr unsigned ReadWriteINTEL = 3u;
- buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::HostAccessINTEL, {ReadWriteINTEL});
+ buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::HostAccessINTEL,
+ {ReadWriteINTEL});
MachineInstrBuilder MIB(*MF, --MIRBuilder.getInsertPt());
addStringImm(GV->getName(), MIB);
}
>From ead8ee95634ed8513c1464b69bb5f7d682fd327b Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Tue, 17 Mar 2026 20:55:47 +0000
Subject: [PATCH 07/29] Handle ASM with multiple outputs.
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 6 +-
llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 79 +++++++++++--------
.../Target/SPIRV/SPIRVPrepareFunctions.cpp | 27 +++++--
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 33 ++++++++
llvm/lib/Target/SPIRV/SPIRVUtils.h | 3 +
.../SPV_INTEL_inline_assembly/inline_asm.ll | 17 ++++
6 files changed, 123 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 131b56e92b8be..b77cc6fc84d29 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -1634,11 +1634,11 @@ Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
if (!Call.isInlineAsm())
return &Call;
- const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
LLVMContext &Ctx = CurrF->getContext();
- Constant *TyC = UndefValue::get(IA->getFunctionType());
- MDString *ConstraintString = MDString::get(Ctx, IA->getConstraintString());
+ Constant *TyC = UndefValue::get(SPIRV::getOriginalFunctionType(Call));
+ MDString *ConstraintString =
+ MDString::get(Ctx, SPIRV::getOriginalAsmConstraints(Call));
SmallVector<Value *> Args = {
buildMD(TyC),
MetadataAsValue::get(Ctx, MDNode::get(Ctx, ConstraintString))};
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index aead1ce735c49..12a326235934e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -41,6 +41,12 @@ void SPIRVPreLegalizer::getAnalysisUsage(AnalysisUsage &AU) const {
MachineFunctionPass::getAnalysisUsage(AU);
}
+static inline void invalidateAndEraseMI(SPIRVGlobalRegistry *GR,
+ MachineInstr *MI) {
+ GR->invalidateMachineInstr(MI);
+ MI->eraseFromParent();
+}
+
static void
addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR,
const SPIRVSubtarget &STI,
@@ -126,13 +132,10 @@ addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR,
if (!MRI.getRegClassOrNull(Reg) && RC)
MRI.setRegClass(Reg, RC);
MRI.replaceRegWith(MI->getOperand(0).getReg(), Reg);
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
- for (MachineInstr *MI : ToEraseComposites) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
+ invalidateAndEraseMI(GR, MI);
}
+ for (MachineInstr *MI : ToEraseComposites)
+ invalidateAndEraseMI(GR, MI);
}
static void foldConstantsIntoIntrinsics(MachineFunction &MF,
@@ -151,10 +154,8 @@ static void foldConstantsIntoIntrinsics(MachineFunction &MF,
}
ToErase.push_back(&MI);
}
- for (MachineInstr *MI : ToErase) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
+ for (MachineInstr *MI : ToErase)
+ invalidateAndEraseMI(GR, MI);
ToErase.clear();
}
}
@@ -235,10 +236,8 @@ static void lowerBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
ToErase.push_back(&MI);
}
}
- for (MachineInstr *MI : ToErase) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
+ for (MachineInstr *MI : ToErase)
+ invalidateAndEraseMI(GR, MI);
}
static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
@@ -279,10 +278,8 @@ static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
}
}
}
- for (MachineInstr *MI : ToErase) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
+ for (MachineInstr *MI : ToErase)
+ invalidateAndEraseMI(GR, MI);
}
// Translating GV, IRTranslator sometimes generates following IR:
@@ -623,8 +620,7 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
auto It = RegsAlreadyAddedToDT.find(MI);
if (It != RegsAlreadyAddedToDT.end())
MRI.replaceRegWith(MI->getOperand(0).getReg(), It->second);
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
+ invalidateAndEraseMI(GR, MI);
}
// Address the case when IRTranslator introduces instructions with new
@@ -759,11 +755,31 @@ insertInlineAsmProcess(MachineFunction &MF, SPIRVGlobalRegistry *GR,
.addUse(AsmReg);
for (unsigned IntrIdx = 3; IntrIdx < I1->getNumOperands(); ++IntrIdx)
AsmCall.addUse(I1->getOperand(IntrIdx).getReg());
+
+ // IRTranslator gets a bit confused when lowering inline ASM with multiple,
+ // outputs (which we have to spoof as single i32 return), and inserts a
+ // spurious COPY & TRUNC as registers are assumed to be i64; we have to
+ // clean that up here to prevent an erroneous cast on a struct to get
+ // lowered into SPIR-V
+ if (FTy->getReturnType()->isStructTy()) {
+ MachineInstr &Copy = *++I2->getIterator();
+ MachineInstr &Trunc = *++Copy.getIterator();
+
+ assert(Copy.isCopy() && Trunc.getOpcode() == TargetOpcode::G_TRUNC &&
+ "Unexpected successors to inline ASM with multiple outputs!");
+
+ Register TruncReg = Trunc.defs().begin()->getReg();
+ MRI.replaceRegWith(TruncReg, DefReg);
+ // for (auto It = ++Trunc.getIterator();
+ // isSpvIntrinsic(*It, Intrinsic::spv_extractv); ++It)
+ // if (It->getOperand(2).getReg() == TruncReg)
+ // It->getOperand(2).setReg(DefReg);
+ invalidateAndEraseMI(GR, &Trunc);
+ invalidateAndEraseMI(GR, &Copy);
+ }
}
- for (MachineInstr *MI : ToProcess) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
+ for (MachineInstr *MI : ToProcess)
+ invalidateAndEraseMI(GR, MI);
}
static void insertInlineAsm(MachineFunction &MF, SPIRVGlobalRegistry *GR,
@@ -830,10 +846,8 @@ static void insertSpirvDecorations(MachineFunction &MF, SPIRVGlobalRegistry *GR,
ToErase.push_back(&MI);
}
}
- for (MachineInstr *MI : ToErase) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
+ for (MachineInstr *MI : ToErase)
+ invalidateAndEraseMI(GR, MI);
}
// LLVM allows the switches to use registers as cases, while SPIR-V required
@@ -883,10 +897,8 @@ static void cleanupHelperInstructions(MachineFunction &MF,
}
}
- for (MachineInstr *MI : ToEraseMI) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
+ for (MachineInstr *MI : ToEraseMI)
+ invalidateAndEraseMI(GR, MI);
}
// Find all usages of G_BLOCK_ADDR in our intrinsics and replace those
@@ -988,8 +1000,7 @@ static void processBlockAddr(MachineFunction &MF, SPIRVGlobalRegistry *GR,
ConstantExpr::getIntToPtr(Replacement, BA->getType()));
BA->destroyConstant();
}
- GR->invalidateMachineInstr(BlockAddrI);
- BlockAddrI->eraseFromParent();
+ invalidateAndEraseMI(GR, BlockAddrI);
}
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 06b39800d2f06..61c517af82972 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -507,7 +507,8 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
static void
addFunctionTypeMutation(NamedMDNode *NMD,
SmallVector<std::pair<int, Type *>> ChangedTys,
- StringRef Name) {
+ StringRef Name,
+ StringRef AsmConstraints = "") {
LLVMContext &Ctx = NMD->getParent()->getContext();
Type *I32Ty = IntegerType::getInt32Ty(Ctx);
@@ -519,8 +520,11 @@ addFunctionTypeMutation(NamedMDNode *NMD,
Ctx, {ConstantAsMetadata::get(ConstantInt::get(I32Ty, CTy.first, true)),
ValueAsMetadata::get(Constant::getNullValue(CTy.second))});
});
+ if (!AsmConstraints.empty())
+ MDArgs.push_back(MDNode::get(Ctx, MDString::get(Ctx, AsmConstraints)));
NMD->addOperand(MDNode::get(Ctx, MDArgs));
}
+
// Returns F if aggregate argument/return types are not present or cloned F
// function with the types replaced by i32 types. The change in types is
// noted in 'spv.cloned_funcs' metadata for later restoration.
@@ -592,9 +596,11 @@ SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) {
return NewF;
}
-// Mutates indirect callsites iff if aggregate argument/return types are present
-// with the types replaced by i32 types. The change in types is noted in
-// 'spv.mutated_callsites' metadata for later restoration.
+// Mutates indirect and inline ASM callsites iff if aggregate argument/return
+// types are present with the types replaced by i32 types. The change in types
+// is noted in 'spv.mutated_callsites' metadata for later restoration. For ASM
+// we also have to mutate the constraint string as IRTranslator tries to handle
+// multiple outputs and expects an aggregate return type in their presence.
bool SPIRVPrepareFunctions::removeAggregateTypesFromCalls(Function *F) {
if (F->isDeclaration() || F->isIntrinsic())
return false;
@@ -643,9 +649,20 @@ bool SPIRVPrepareFunctions::removeAggregateTypesFromCalls(Function *F) {
CB->setName("spv.named_mutated_callsite." + F->getName() + "." +
CB->getName());
+ StringRef MaybeConstraints;
+ if (auto *ASM = dyn_cast<InlineAsm>(CB->getCalledOperand())) {
+ MaybeConstraints = ASM->getConstraintString();
+ // We should only have one =r return for the made up ASM type.
+ CB->setCalledOperand(InlineAsm::get(
+ NewFnTy, ASM->getAsmString(),
+ ASM->getConstraintString().substr(MaybeConstraints.find_last_of('=')),
+ ASM->hasSideEffects(), ASM->isAlignStack(), ASM->getDialect(),
+ ASM->canThrow()));
+ }
+
addFunctionTypeMutation(
F->getParent()->getOrInsertNamedMetadata("spv.mutated_callsites"),
- std::move(ChangedTypes), CB->getName());
+ std::move(ChangedTypes), CB->getName(), MaybeConstraints);
}
for (auto &&[CB, NewFTy] : Calls) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 4a25886614098..25726b5d0ea9d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -79,6 +79,32 @@ static FunctionType *extractFunctionTypeFromMetadata(NamedMDNode *NMD,
return FunctionType::get(RetTy, PTys, FTy->isVarArg());
}
+static StringRef extractAsmConstraintsFromMetadata(NamedMDNode *NMD,
+ StringRef Constraints,
+ StringRef Name) {
+ // TODO: unify the extractors.
+ if (!NMD)
+ return Constraints;
+
+ auto It = find_if(NMD->operands(), [Name](MDNode *N) {
+ if (auto *MDS = dyn_cast_or_null<MDString>(N->getOperand(0)))
+ return MDS->getString() == Name;
+ return false;
+ });
+
+ if (It == NMD->op_end())
+ return Constraints;
+
+ // By convention, the constraints string is stored in the final MD operand.
+ MDNode *MD = dyn_cast<MDNode>((*It)->getOperand((*It)->getNumOperands() - 1));
+ assert(MD && "MDNode operand is expected");
+
+ if (auto *MDS = dyn_cast<MDString>(MD->getOperand(0)))
+ Constraints = MDS->getString();
+
+ return Constraints;
+}
+
FunctionType *getOriginalFunctionType(const Function &F) {
return extractFunctionTypeFromMetadata(
F.getParent()->getNamedMetadata("spv.cloned_funcs"), F.getFunctionType(),
@@ -90,6 +116,13 @@ FunctionType *getOriginalFunctionType(const CallBase &CB) {
CB.getModule()->getNamedMetadata("spv.mutated_callsites"),
CB.getFunctionType(), CB.getName());
}
+
+StringRef getOriginalAsmConstraints(const CallBase &CB) {
+ return extractAsmConstraintsFromMetadata(
+ CB.getModule()->getNamedMetadata("spv.mutated_callsites"),
+ cast<InlineAsm>(CB.getCalledOperand())->getConstraintString(),
+ CB.getName());
+}
} // Namespace SPIRV
// The following functions are used to add these string literals as a series of
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index d541ead5ac22c..c7b807f97d13c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -167,6 +167,9 @@ struct FPFastMathDefaultInfoVector
// during the translation to cope with aggregate flattening etc.
FunctionType *getOriginalFunctionType(const Function &F);
FunctionType *getOriginalFunctionType(const CallBase &CB);
+// This handles retrieving the original ASM constraints, which we had to spoof
+// into having a single output.
+StringRef getOriginalAsmConstraints(const CallBase &CB);
} // namespace SPIRV
// Add the given string as a series of integer operand, inserting null
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
index 91286d5bf32e8..805669e6eae3b 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
@@ -16,6 +16,7 @@
; CHECK-DAG: %[[#HalfTy:]] = OpTypeFloat 16
; CHECK-DAG: %[[#FloatTy:]] = OpTypeFloat 32
; CHECK-DAG: %[[#DoubleTy:]] = OpTypeFloat 64
+; CHECK-DAG: %[[#StructTy:]] = OpTypeStruct %[[#Int32Ty]] %[[#FloatTy]] %[[#HalfTy]]
; CHECK-DAG: OpTypeFunction %[[#VoidTy]] %[[#]] %[[#]] %[[#]] %[[#Int64Ty]]
; CHECK-DAG: %[[#Fun1Ty:]] = OpTypeFunction %[[#VoidTy]]
@@ -26,6 +27,7 @@
; CHECK-DAG: %[[#Fun6Ty:]] = OpTypeFunction %[[#Int8Ty]] %[[#FloatTy]] %[[#Int32Ty]] %[[#Int8Ty]]
; CHECK-DAG: %[[#Fun7Ty:]] = OpTypeFunction %[[#Int64Ty]] %[[#Int64Ty]] %[[#Int32Ty]] %[[#Int8Ty]]
; CHECK-DAG: %[[#Fun8Ty:]] = OpTypeFunction %[[#VoidTy]] %[[#Int32Ty]] %[[#DoubleTy]]
+; CHECK-DAG: %[[#Fun9Ty:]] = OpTypeFunction %[[#StructTy]] %[[#Int32Ty]] %[[#FloatTy]] %[[#HalfTy]]
; CHECK-DAG: %[[#Const2:]] = OpConstant %[[#FloatTy]] 2
; CHECK-DAG: %[[#Const123:]] = OpConstant %[[#Int32Ty]] 123
@@ -45,6 +47,7 @@
; CHECK-DAG: %[[#Asm9:]] = OpAsmINTEL %[[#Int64Ty]] %[[#Fun7Ty]] %[[#Dialect]] "icmdext $0 $3 $1 $2" "=r,r,r,r"
; CHECK-DAG: %[[#Asm10:]] = OpAsmINTEL %[[#VoidTy]] %[[#Fun8Ty]] %[[#Dialect]] "constcmd $0 $1" "r,r"
; CHECK-DAG: %[[#Asm11:]] = OpAsmINTEL %[[#VoidTy]] %[[#Fun8Ty]] %[[#Dialect]] "constcmd $0 $1" "i,i"
+; CHECK-DAG: %[[#Asm12:]] = OpAsmINTEL %[[#StructTy]] %[[#Fun9Ty]] %[[#Dialect]] "cmdext $0 $4 $5\n cmdext $2 $5 $6\n cmdext $3 $4 $6" "=&r,=&r,=&r,r,r,r"
; CHECK-NO: OpAsmINTEL
; CHECK: OpFunction
@@ -59,8 +62,14 @@
; CHECK: OpAsmCallINTEL %[[#Int64Ty]] %[[#Asm9]] %[[#]] %[[#]] %[[#]]
; CHECK: OpAsmCallINTEL %[[#VoidTy]] %[[#Asm10]] %[[#Const123]] %[[#Const42]]
; CHECK: OpAsmCallINTEL %[[#VoidTy]] %[[#Asm11]] %[[#Const123]] %[[#Const42]]
+; CHECK: %[[#StructRet:]] = OpAsmCallINTEL %[[#StructTy]] %[[#Asm12]]
+; CHECK-NEXT: OpCompositeExtract %[[#Int32Ty]] %[[#StructRet]] 0
+; CHECK-NEXT: OpCompositeExtract %[[#FloatTy]] %[[#StructRet]] 1
+; CHECK-NEXT: OpCompositeExtract %[[#HalfTy]] %[[#StructRet]] 2
; CHECK-NO: OpAsmCallINTEL
+target triple = "spirv64-unknown-unknown"
+
define spir_kernel void @foo(ptr addrspace(1) %_arg_int, ptr addrspace(1) %_arg_float, ptr addrspace(1) %_arg_half, i64 %_lng) {
%i1 = load i32, ptr addrspace(1) %_arg_int
%i2 = load i8, ptr addrspace(1) %_arg_int
@@ -89,5 +98,13 @@ define spir_kernel void @foo(ptr addrspace(1) %_arg_int, ptr addrspace(1) %_arg_
; inline asm: constant arguments, misc constraints
call void asm "constcmd $0 $1", "r,r"(i32 123, double 42.0)
call void asm "constcmd $0 $1", "i,i"(i32 123, double 42.0)
+ ; inline asm: multiple outputs, hence aggregate return
+ %res_struct = call { i32, float, half } asm sideeffect "cmdext $0 $4 $5\0A cmdext $2 $5 $6\0A cmdext $3 $4 $6", "=&r,=&r,=&r,r,r,r"(i32 %i1, float %f1, half %h1)
+ %asmresult = extractvalue { i32, float, half } %res_struct, 0
+ %asmresult3 = extractvalue { i32, float, half } %res_struct, 1
+ %asmresult4 = extractvalue { i32, float, half } %res_struct, 2
+ store i32 %asmresult, ptr addrspace(1) %_arg_int
+ store float %asmresult3, ptr addrspace(1) %_arg_float
+ store half %asmresult4, ptr addrspace(1) %_arg_half
ret void
}
>From 75a092f4c23670693b80cd8883ce8c7ad1c4ea85 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Tue, 17 Mar 2026 21:03:40 +0000
Subject: [PATCH 08/29] Fix formatting.
---
llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp | 3 +--
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 6 +++---
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 61c517af82972..f2c436798083f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -507,8 +507,7 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
static void
addFunctionTypeMutation(NamedMDNode *NMD,
SmallVector<std::pair<int, Type *>> ChangedTys,
- StringRef Name,
- StringRef AsmConstraints = "") {
+ StringRef Name, StringRef AsmConstraints = "") {
LLVMContext &Ctx = NMD->getParent()->getContext();
Type *I32Ty = IntegerType::getInt32Ty(Ctx);
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 25726b5d0ea9d..be91650b59654 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -119,9 +119,9 @@ FunctionType *getOriginalFunctionType(const CallBase &CB) {
StringRef getOriginalAsmConstraints(const CallBase &CB) {
return extractAsmConstraintsFromMetadata(
- CB.getModule()->getNamedMetadata("spv.mutated_callsites"),
- cast<InlineAsm>(CB.getCalledOperand())->getConstraintString(),
- CB.getName());
+ CB.getModule()->getNamedMetadata("spv.mutated_callsites"),
+ cast<InlineAsm>(CB.getCalledOperand())->getConstraintString(),
+ CB.getName());
}
} // Namespace SPIRV
>From 341c285258444edb4b14b6452947b0272311ce1e Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Wed, 18 Mar 2026 12:43:14 +0000
Subject: [PATCH 09/29] Update llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Co-authored-by: Juan Manuel Martinez Caamaño <jmartinezcaamao at gmail.com>
---
llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index f2c436798083f..350fee89614ef 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -654,7 +654,7 @@ bool SPIRVPrepareFunctions::removeAggregateTypesFromCalls(Function *F) {
// We should only have one =r return for the made up ASM type.
CB->setCalledOperand(InlineAsm::get(
NewFnTy, ASM->getAsmString(),
- ASM->getConstraintString().substr(MaybeConstraints.find_last_of('=')),
+ MaybeConstraints.substr(MaybeConstraints.find_last_of('=')),
ASM->hasSideEffects(), ASM->isAlignStack(), ASM->getDialect(),
ASM->canThrow()));
}
>From e0f029189cf086619337ca56f87541d2dfd863c0 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Thu, 19 Mar 2026 00:57:37 +0000
Subject: [PATCH 10/29] Remove leftover.
---
llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 4 ----
1 file changed, 4 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 12a326235934e..456a618effdc2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -770,10 +770,6 @@ insertInlineAsmProcess(MachineFunction &MF, SPIRVGlobalRegistry *GR,
Register TruncReg = Trunc.defs().begin()->getReg();
MRI.replaceRegWith(TruncReg, DefReg);
- // for (auto It = ++Trunc.getIterator();
- // isSpvIntrinsic(*It, Intrinsic::spv_extractv); ++It)
- // if (It->getOperand(2).getReg() == TruncReg)
- // It->getOperand(2).setReg(DefReg);
invalidateAndEraseMI(GR, &Trunc);
invalidateAndEraseMI(GR, &Copy);
}
>From d440d1aaab3d3c0fd3b26557099cf19f3597fae7 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Thu, 19 Mar 2026 00:59:02 +0000
Subject: [PATCH 11/29] Remove more leftovers.
---
.../SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
index 805669e6eae3b..ff6b6dec42ff3 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
@@ -68,8 +68,6 @@
; CHECK-NEXT: OpCompositeExtract %[[#HalfTy]] %[[#StructRet]] 2
; CHECK-NO: OpAsmCallINTEL
-target triple = "spirv64-unknown-unknown"
-
define spir_kernel void @foo(ptr addrspace(1) %_arg_int, ptr addrspace(1) %_arg_float, ptr addrspace(1) %_arg_half, i64 %_lng) {
%i1 = load i32, ptr addrspace(1) %_arg_int
%i2 = load i8, ptr addrspace(1) %_arg_int
>From ae59e40b87874318e03a54f5e0b8f430f82ef3ce Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Fri, 20 Mar 2026 23:00:10 +0200
Subject: [PATCH 12/29] Generalise constraint string mutation to handle
indirect operands.
---
.../Target/SPIRV/SPIRVPrepareFunctions.cpp | 29 ++++++++++++++-----
1 file changed, 21 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 350fee89614ef..1533e0f8e472d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -595,6 +595,21 @@ SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) {
return NewF;
}
+static std::string fixMultiOutputConstraintString(StringRef Constraints) {
+ // We should only have one =r return for the made up ASM type.
+ SmallVector<StringRef> Tmp;
+ SplitString(Constraints, Tmp, ",");
+ std::string SafeConstraints("=r,");
+ for (unsigned I = 0u; I != Tmp.size() - 1; ++I) {
+ if (Tmp[I].starts_with('=') && isalnum(Tmp[I][1]))
+ continue;
+ SafeConstraints.append(Tmp[I]).append({','});
+ }
+ SafeConstraints.append(Tmp.back());
+
+ return SafeConstraints;
+}
+
// Mutates indirect and inline ASM callsites iff if aggregate argument/return
// types are present with the types replaced by i32 types. The change in types
// is noted in 'spv.mutated_callsites' metadata for later restoration. For ASM
@@ -648,20 +663,18 @@ bool SPIRVPrepareFunctions::removeAggregateTypesFromCalls(Function *F) {
CB->setName("spv.named_mutated_callsite." + F->getName() + "." +
CB->getName());
- StringRef MaybeConstraints;
+ std::string Constraints;
if (auto *ASM = dyn_cast<InlineAsm>(CB->getCalledOperand())) {
- MaybeConstraints = ASM->getConstraintString();
- // We should only have one =r return for the made up ASM type.
+ Constraints = fixMultiOutputConstraintString(ASM->getConstraintString());
+
CB->setCalledOperand(InlineAsm::get(
- NewFnTy, ASM->getAsmString(),
- MaybeConstraints.substr(MaybeConstraints.find_last_of('=')),
- ASM->hasSideEffects(), ASM->isAlignStack(), ASM->getDialect(),
- ASM->canThrow()));
+ NewFnTy, ASM->getAsmString(), Constraints, ASM->hasSideEffects(),
+ ASM->isAlignStack(), ASM->getDialect(), ASM->canThrow()));
}
addFunctionTypeMutation(
F->getParent()->getOrInsertNamedMetadata("spv.mutated_callsites"),
- std::move(ChangedTypes), CB->getName(), MaybeConstraints);
+ std::move(ChangedTypes), CB->getName(), Constraints);
}
for (auto &&[CB, NewFTy] : Calls) {
>From 23c5cafecae2e38aafbe19b724d5de3980529bba Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Fri, 20 Mar 2026 23:04:18 +0200
Subject: [PATCH 13/29] Handle memory constraints.
---
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 2fcb71939322a..4469e3943204c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -136,8 +136,12 @@ TargetLowering::ConstraintType
SPIRVTargetLowering::getConstraintType(StringRef Constraint) const {
// SPIR-V represents inline assembly via OpAsmINTEL where constraints are
// passed through as literals defined by client API. Return C_RegisterClass
- // for any constraint since SPIR-V does not distinguish between register,
- // immediate, or memory operands at this level.
+ // for non-memory constraints since SPIR-V does not distinguish between register,
+ // immediate, or memory operands at this level. We do have to return C_Memory
+ // for memory constraints as otherwise IRTranslator gets confused trying to
+ // allocate registers for them.
+ if (Constraint == "m")
+ return C_Memory;
return C_RegisterClass;
}
>From d4ef5c16d1c1baffb640a69ff5daef7d812e7fb8 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Tue, 17 Mar 2026 20:55:47 +0000
Subject: [PATCH 14/29] Handle ASM with multiple outputs.
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 6 +-
llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 79 +++++++++++--------
.../Target/SPIRV/SPIRVPrepareFunctions.cpp | 27 +++++--
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 33 ++++++++
llvm/lib/Target/SPIRV/SPIRVUtils.h | 3 +
.../SPV_INTEL_inline_assembly/inline_asm.ll | 17 ++++
6 files changed, 123 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 9c2af100ad64e..3857c7248ca54 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -1661,11 +1661,11 @@ Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
if (!Call.isInlineAsm())
return &Call;
- const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
LLVMContext &Ctx = CurrF->getContext();
- Constant *TyC = UndefValue::get(IA->getFunctionType());
- MDString *ConstraintString = MDString::get(Ctx, IA->getConstraintString());
+ Constant *TyC = UndefValue::get(SPIRV::getOriginalFunctionType(Call));
+ MDString *ConstraintString =
+ MDString::get(Ctx, SPIRV::getOriginalAsmConstraints(Call));
SmallVector<Value *> Args = {
buildMD(TyC),
MetadataAsValue::get(Ctx, MDNode::get(Ctx, ConstraintString))};
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index aead1ce735c49..12a326235934e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -41,6 +41,12 @@ void SPIRVPreLegalizer::getAnalysisUsage(AnalysisUsage &AU) const {
MachineFunctionPass::getAnalysisUsage(AU);
}
+static inline void invalidateAndEraseMI(SPIRVGlobalRegistry *GR,
+ MachineInstr *MI) {
+ GR->invalidateMachineInstr(MI);
+ MI->eraseFromParent();
+}
+
static void
addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR,
const SPIRVSubtarget &STI,
@@ -126,13 +132,10 @@ addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR,
if (!MRI.getRegClassOrNull(Reg) && RC)
MRI.setRegClass(Reg, RC);
MRI.replaceRegWith(MI->getOperand(0).getReg(), Reg);
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
- for (MachineInstr *MI : ToEraseComposites) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
+ invalidateAndEraseMI(GR, MI);
}
+ for (MachineInstr *MI : ToEraseComposites)
+ invalidateAndEraseMI(GR, MI);
}
static void foldConstantsIntoIntrinsics(MachineFunction &MF,
@@ -151,10 +154,8 @@ static void foldConstantsIntoIntrinsics(MachineFunction &MF,
}
ToErase.push_back(&MI);
}
- for (MachineInstr *MI : ToErase) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
+ for (MachineInstr *MI : ToErase)
+ invalidateAndEraseMI(GR, MI);
ToErase.clear();
}
}
@@ -235,10 +236,8 @@ static void lowerBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
ToErase.push_back(&MI);
}
}
- for (MachineInstr *MI : ToErase) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
+ for (MachineInstr *MI : ToErase)
+ invalidateAndEraseMI(GR, MI);
}
static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
@@ -279,10 +278,8 @@ static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
}
}
}
- for (MachineInstr *MI : ToErase) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
+ for (MachineInstr *MI : ToErase)
+ invalidateAndEraseMI(GR, MI);
}
// Translating GV, IRTranslator sometimes generates following IR:
@@ -623,8 +620,7 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
auto It = RegsAlreadyAddedToDT.find(MI);
if (It != RegsAlreadyAddedToDT.end())
MRI.replaceRegWith(MI->getOperand(0).getReg(), It->second);
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
+ invalidateAndEraseMI(GR, MI);
}
// Address the case when IRTranslator introduces instructions with new
@@ -759,11 +755,31 @@ insertInlineAsmProcess(MachineFunction &MF, SPIRVGlobalRegistry *GR,
.addUse(AsmReg);
for (unsigned IntrIdx = 3; IntrIdx < I1->getNumOperands(); ++IntrIdx)
AsmCall.addUse(I1->getOperand(IntrIdx).getReg());
+
+ // IRTranslator gets a bit confused when lowering inline ASM with multiple,
+ // outputs (which we have to spoof as single i32 return), and inserts a
+ // spurious COPY & TRUNC as registers are assumed to be i64; we have to
+ // clean that up here to prevent an erroneous cast on a struct to get
+ // lowered into SPIR-V
+ if (FTy->getReturnType()->isStructTy()) {
+ MachineInstr &Copy = *++I2->getIterator();
+ MachineInstr &Trunc = *++Copy.getIterator();
+
+ assert(Copy.isCopy() && Trunc.getOpcode() == TargetOpcode::G_TRUNC &&
+ "Unexpected successors to inline ASM with multiple outputs!");
+
+ Register TruncReg = Trunc.defs().begin()->getReg();
+ MRI.replaceRegWith(TruncReg, DefReg);
+ // for (auto It = ++Trunc.getIterator();
+ // isSpvIntrinsic(*It, Intrinsic::spv_extractv); ++It)
+ // if (It->getOperand(2).getReg() == TruncReg)
+ // It->getOperand(2).setReg(DefReg);
+ invalidateAndEraseMI(GR, &Trunc);
+ invalidateAndEraseMI(GR, &Copy);
+ }
}
- for (MachineInstr *MI : ToProcess) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
+ for (MachineInstr *MI : ToProcess)
+ invalidateAndEraseMI(GR, MI);
}
static void insertInlineAsm(MachineFunction &MF, SPIRVGlobalRegistry *GR,
@@ -830,10 +846,8 @@ static void insertSpirvDecorations(MachineFunction &MF, SPIRVGlobalRegistry *GR,
ToErase.push_back(&MI);
}
}
- for (MachineInstr *MI : ToErase) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
+ for (MachineInstr *MI : ToErase)
+ invalidateAndEraseMI(GR, MI);
}
// LLVM allows the switches to use registers as cases, while SPIR-V required
@@ -883,10 +897,8 @@ static void cleanupHelperInstructions(MachineFunction &MF,
}
}
- for (MachineInstr *MI : ToEraseMI) {
- GR->invalidateMachineInstr(MI);
- MI->eraseFromParent();
- }
+ for (MachineInstr *MI : ToEraseMI)
+ invalidateAndEraseMI(GR, MI);
}
// Find all usages of G_BLOCK_ADDR in our intrinsics and replace those
@@ -988,8 +1000,7 @@ static void processBlockAddr(MachineFunction &MF, SPIRVGlobalRegistry *GR,
ConstantExpr::getIntToPtr(Replacement, BA->getType()));
BA->destroyConstant();
}
- GR->invalidateMachineInstr(BlockAddrI);
- BlockAddrI->eraseFromParent();
+ invalidateAndEraseMI(GR, BlockAddrI);
}
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index a3b44ad6d31d5..9645f6a6e802d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -510,7 +510,8 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
static void
addFunctionTypeMutation(NamedMDNode *NMD,
SmallVector<std::pair<int, Type *>> ChangedTys,
- StringRef Name) {
+ StringRef Name,
+ StringRef AsmConstraints = "") {
LLVMContext &Ctx = NMD->getParent()->getContext();
Type *I32Ty = IntegerType::getInt32Ty(Ctx);
@@ -522,8 +523,11 @@ addFunctionTypeMutation(NamedMDNode *NMD,
Ctx, {ConstantAsMetadata::get(ConstantInt::get(I32Ty, CTy.first, true)),
ValueAsMetadata::get(Constant::getNullValue(CTy.second))});
});
+ if (!AsmConstraints.empty())
+ MDArgs.push_back(MDNode::get(Ctx, MDString::get(Ctx, AsmConstraints)));
NMD->addOperand(MDNode::get(Ctx, MDArgs));
}
+
// Returns F if aggregate argument/return types are not present or cloned F
// function with the types replaced by i32 types. The change in types is
// noted in 'spv.cloned_funcs' metadata for later restoration.
@@ -595,9 +599,11 @@ SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) {
return NewF;
}
-// Mutates indirect callsites iff if aggregate argument/return types are present
-// with the types replaced by i32 types. The change in types is noted in
-// 'spv.mutated_callsites' metadata for later restoration.
+// Mutates indirect and inline ASM callsites iff if aggregate argument/return
+// types are present with the types replaced by i32 types. The change in types
+// is noted in 'spv.mutated_callsites' metadata for later restoration. For ASM
+// we also have to mutate the constraint string as IRTranslator tries to handle
+// multiple outputs and expects an aggregate return type in their presence.
bool SPIRVPrepareFunctions::removeAggregateTypesFromCalls(Function *F) {
if (F->isDeclaration() || F->isIntrinsic())
return false;
@@ -646,9 +652,20 @@ bool SPIRVPrepareFunctions::removeAggregateTypesFromCalls(Function *F) {
CB->setName("spv.named_mutated_callsite." + F->getName() + "." +
CB->getName());
+ StringRef MaybeConstraints;
+ if (auto *ASM = dyn_cast<InlineAsm>(CB->getCalledOperand())) {
+ MaybeConstraints = ASM->getConstraintString();
+ // We should only have one =r return for the made up ASM type.
+ CB->setCalledOperand(InlineAsm::get(
+ NewFnTy, ASM->getAsmString(),
+ ASM->getConstraintString().substr(MaybeConstraints.find_last_of('=')),
+ ASM->hasSideEffects(), ASM->isAlignStack(), ASM->getDialect(),
+ ASM->canThrow()));
+ }
+
addFunctionTypeMutation(
F->getParent()->getOrInsertNamedMetadata("spv.mutated_callsites"),
- std::move(ChangedTypes), CB->getName());
+ std::move(ChangedTypes), CB->getName(), MaybeConstraints);
}
for (auto &&[CB, NewFTy] : Calls) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index f8e3e27ca289b..61eaf08afbd81 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -79,6 +79,32 @@ static FunctionType *extractFunctionTypeFromMetadata(NamedMDNode *NMD,
return FunctionType::get(RetTy, PTys, FTy->isVarArg());
}
+static StringRef extractAsmConstraintsFromMetadata(NamedMDNode *NMD,
+ StringRef Constraints,
+ StringRef Name) {
+ // TODO: unify the extractors.
+ if (!NMD)
+ return Constraints;
+
+ auto It = find_if(NMD->operands(), [Name](MDNode *N) {
+ if (auto *MDS = dyn_cast_or_null<MDString>(N->getOperand(0)))
+ return MDS->getString() == Name;
+ return false;
+ });
+
+ if (It == NMD->op_end())
+ return Constraints;
+
+ // By convention, the constraints string is stored in the final MD operand.
+ MDNode *MD = dyn_cast<MDNode>((*It)->getOperand((*It)->getNumOperands() - 1));
+ assert(MD && "MDNode operand is expected");
+
+ if (auto *MDS = dyn_cast<MDString>(MD->getOperand(0)))
+ Constraints = MDS->getString();
+
+ return Constraints;
+}
+
FunctionType *getOriginalFunctionType(const Function &F) {
return extractFunctionTypeFromMetadata(
F.getParent()->getNamedMetadata("spv.cloned_funcs"), F.getFunctionType(),
@@ -90,6 +116,13 @@ FunctionType *getOriginalFunctionType(const CallBase &CB) {
CB.getModule()->getNamedMetadata("spv.mutated_callsites"),
CB.getFunctionType(), CB.getName());
}
+
+StringRef getOriginalAsmConstraints(const CallBase &CB) {
+ return extractAsmConstraintsFromMetadata(
+ CB.getModule()->getNamedMetadata("spv.mutated_callsites"),
+ cast<InlineAsm>(CB.getCalledOperand())->getConstraintString(),
+ CB.getName());
+}
} // Namespace SPIRV
// The following functions are used to add these string literals as a series of
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index d541ead5ac22c..c7b807f97d13c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -167,6 +167,9 @@ struct FPFastMathDefaultInfoVector
// during the translation to cope with aggregate flattening etc.
FunctionType *getOriginalFunctionType(const Function &F);
FunctionType *getOriginalFunctionType(const CallBase &CB);
+// This handles retrieving the original ASM constraints, which we had to spoof
+// into having a single output.
+StringRef getOriginalAsmConstraints(const CallBase &CB);
} // namespace SPIRV
// Add the given string as a series of integer operand, inserting null
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
index 91286d5bf32e8..805669e6eae3b 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
@@ -16,6 +16,7 @@
; CHECK-DAG: %[[#HalfTy:]] = OpTypeFloat 16
; CHECK-DAG: %[[#FloatTy:]] = OpTypeFloat 32
; CHECK-DAG: %[[#DoubleTy:]] = OpTypeFloat 64
+; CHECK-DAG: %[[#StructTy:]] = OpTypeStruct %[[#Int32Ty]] %[[#FloatTy]] %[[#HalfTy]]
; CHECK-DAG: OpTypeFunction %[[#VoidTy]] %[[#]] %[[#]] %[[#]] %[[#Int64Ty]]
; CHECK-DAG: %[[#Fun1Ty:]] = OpTypeFunction %[[#VoidTy]]
@@ -26,6 +27,7 @@
; CHECK-DAG: %[[#Fun6Ty:]] = OpTypeFunction %[[#Int8Ty]] %[[#FloatTy]] %[[#Int32Ty]] %[[#Int8Ty]]
; CHECK-DAG: %[[#Fun7Ty:]] = OpTypeFunction %[[#Int64Ty]] %[[#Int64Ty]] %[[#Int32Ty]] %[[#Int8Ty]]
; CHECK-DAG: %[[#Fun8Ty:]] = OpTypeFunction %[[#VoidTy]] %[[#Int32Ty]] %[[#DoubleTy]]
+; CHECK-DAG: %[[#Fun9Ty:]] = OpTypeFunction %[[#StructTy]] %[[#Int32Ty]] %[[#FloatTy]] %[[#HalfTy]]
; CHECK-DAG: %[[#Const2:]] = OpConstant %[[#FloatTy]] 2
; CHECK-DAG: %[[#Const123:]] = OpConstant %[[#Int32Ty]] 123
@@ -45,6 +47,7 @@
; CHECK-DAG: %[[#Asm9:]] = OpAsmINTEL %[[#Int64Ty]] %[[#Fun7Ty]] %[[#Dialect]] "icmdext $0 $3 $1 $2" "=r,r,r,r"
; CHECK-DAG: %[[#Asm10:]] = OpAsmINTEL %[[#VoidTy]] %[[#Fun8Ty]] %[[#Dialect]] "constcmd $0 $1" "r,r"
; CHECK-DAG: %[[#Asm11:]] = OpAsmINTEL %[[#VoidTy]] %[[#Fun8Ty]] %[[#Dialect]] "constcmd $0 $1" "i,i"
+; CHECK-DAG: %[[#Asm12:]] = OpAsmINTEL %[[#StructTy]] %[[#Fun9Ty]] %[[#Dialect]] "cmdext $0 $4 $5\n cmdext $2 $5 $6\n cmdext $3 $4 $6" "=&r,=&r,=&r,r,r,r"
; CHECK-NO: OpAsmINTEL
; CHECK: OpFunction
@@ -59,8 +62,14 @@
; CHECK: OpAsmCallINTEL %[[#Int64Ty]] %[[#Asm9]] %[[#]] %[[#]] %[[#]]
; CHECK: OpAsmCallINTEL %[[#VoidTy]] %[[#Asm10]] %[[#Const123]] %[[#Const42]]
; CHECK: OpAsmCallINTEL %[[#VoidTy]] %[[#Asm11]] %[[#Const123]] %[[#Const42]]
+; CHECK: %[[#StructRet:]] = OpAsmCallINTEL %[[#StructTy]] %[[#Asm12]]
+; CHECK-NEXT: OpCompositeExtract %[[#Int32Ty]] %[[#StructRet]] 0
+; CHECK-NEXT: OpCompositeExtract %[[#FloatTy]] %[[#StructRet]] 1
+; CHECK-NEXT: OpCompositeExtract %[[#HalfTy]] %[[#StructRet]] 2
; CHECK-NO: OpAsmCallINTEL
+target triple = "spirv64-unknown-unknown"
+
define spir_kernel void @foo(ptr addrspace(1) %_arg_int, ptr addrspace(1) %_arg_float, ptr addrspace(1) %_arg_half, i64 %_lng) {
%i1 = load i32, ptr addrspace(1) %_arg_int
%i2 = load i8, ptr addrspace(1) %_arg_int
@@ -89,5 +98,13 @@ define spir_kernel void @foo(ptr addrspace(1) %_arg_int, ptr addrspace(1) %_arg_
; inline asm: constant arguments, misc constraints
call void asm "constcmd $0 $1", "r,r"(i32 123, double 42.0)
call void asm "constcmd $0 $1", "i,i"(i32 123, double 42.0)
+ ; inline asm: multiple outputs, hence aggregate return
+ %res_struct = call { i32, float, half } asm sideeffect "cmdext $0 $4 $5\0A cmdext $2 $5 $6\0A cmdext $3 $4 $6", "=&r,=&r,=&r,r,r,r"(i32 %i1, float %f1, half %h1)
+ %asmresult = extractvalue { i32, float, half } %res_struct, 0
+ %asmresult3 = extractvalue { i32, float, half } %res_struct, 1
+ %asmresult4 = extractvalue { i32, float, half } %res_struct, 2
+ store i32 %asmresult, ptr addrspace(1) %_arg_int
+ store float %asmresult3, ptr addrspace(1) %_arg_float
+ store half %asmresult4, ptr addrspace(1) %_arg_half
ret void
}
>From 26179873668ad70298b4b2928bee36544ab9a9dd Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Tue, 17 Mar 2026 21:03:40 +0000
Subject: [PATCH 15/29] Fix formatting.
---
llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp | 3 +--
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 6 +++---
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 9645f6a6e802d..192c6cd4824fe 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -510,8 +510,7 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
static void
addFunctionTypeMutation(NamedMDNode *NMD,
SmallVector<std::pair<int, Type *>> ChangedTys,
- StringRef Name,
- StringRef AsmConstraints = "") {
+ StringRef Name, StringRef AsmConstraints = "") {
LLVMContext &Ctx = NMD->getParent()->getContext();
Type *I32Ty = IntegerType::getInt32Ty(Ctx);
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 61eaf08afbd81..4dfcad79d1914 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -119,9 +119,9 @@ FunctionType *getOriginalFunctionType(const CallBase &CB) {
StringRef getOriginalAsmConstraints(const CallBase &CB) {
return extractAsmConstraintsFromMetadata(
- CB.getModule()->getNamedMetadata("spv.mutated_callsites"),
- cast<InlineAsm>(CB.getCalledOperand())->getConstraintString(),
- CB.getName());
+ CB.getModule()->getNamedMetadata("spv.mutated_callsites"),
+ cast<InlineAsm>(CB.getCalledOperand())->getConstraintString(),
+ CB.getName());
}
} // Namespace SPIRV
>From a0076bd1d52c0d1b5329c5c1740239276d230ece Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Wed, 18 Mar 2026 12:43:14 +0000
Subject: [PATCH 16/29] Update llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Co-authored-by: Juan Manuel Martinez Caamaño <jmartinezcaamao at gmail.com>
---
llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 192c6cd4824fe..4a76754b8a4f1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -657,7 +657,7 @@ bool SPIRVPrepareFunctions::removeAggregateTypesFromCalls(Function *F) {
// We should only have one =r return for the made up ASM type.
CB->setCalledOperand(InlineAsm::get(
NewFnTy, ASM->getAsmString(),
- ASM->getConstraintString().substr(MaybeConstraints.find_last_of('=')),
+ MaybeConstraints.substr(MaybeConstraints.find_last_of('=')),
ASM->hasSideEffects(), ASM->isAlignStack(), ASM->getDialect(),
ASM->canThrow()));
}
>From 96712f4a88286cd5193edb8544443f3742aefa98 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Thu, 19 Mar 2026 00:57:37 +0000
Subject: [PATCH 17/29] Remove leftover.
---
llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 4 ----
1 file changed, 4 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 12a326235934e..456a618effdc2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -770,10 +770,6 @@ insertInlineAsmProcess(MachineFunction &MF, SPIRVGlobalRegistry *GR,
Register TruncReg = Trunc.defs().begin()->getReg();
MRI.replaceRegWith(TruncReg, DefReg);
- // for (auto It = ++Trunc.getIterator();
- // isSpvIntrinsic(*It, Intrinsic::spv_extractv); ++It)
- // if (It->getOperand(2).getReg() == TruncReg)
- // It->getOperand(2).setReg(DefReg);
invalidateAndEraseMI(GR, &Trunc);
invalidateAndEraseMI(GR, &Copy);
}
>From 6dba483255cdfb96e32fc3b18dca9ac8f120e6cb Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Thu, 19 Mar 2026 00:59:02 +0000
Subject: [PATCH 18/29] Remove more leftovers.
---
.../SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
index 805669e6eae3b..ff6b6dec42ff3 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
@@ -68,8 +68,6 @@
; CHECK-NEXT: OpCompositeExtract %[[#HalfTy]] %[[#StructRet]] 2
; CHECK-NO: OpAsmCallINTEL
-target triple = "spirv64-unknown-unknown"
-
define spir_kernel void @foo(ptr addrspace(1) %_arg_int, ptr addrspace(1) %_arg_float, ptr addrspace(1) %_arg_half, i64 %_lng) {
%i1 = load i32, ptr addrspace(1) %_arg_int
%i2 = load i8, ptr addrspace(1) %_arg_int
>From c711eaf921b4b4a71240585a0faea0eb252d1fea Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Fri, 20 Mar 2026 23:00:10 +0200
Subject: [PATCH 19/29] Generalise constraint string mutation to handle
indirect operands.
---
.../Target/SPIRV/SPIRVPrepareFunctions.cpp | 29 ++++++++++++++-----
1 file changed, 21 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 4a76754b8a4f1..813f943afb5c4 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -598,6 +598,21 @@ SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) {
return NewF;
}
+static std::string fixMultiOutputConstraintString(StringRef Constraints) {
+ // We should only have one =r return for the made up ASM type.
+ SmallVector<StringRef> Tmp;
+ SplitString(Constraints, Tmp, ",");
+ std::string SafeConstraints("=r,");
+ for (unsigned I = 0u; I != Tmp.size() - 1; ++I) {
+ if (Tmp[I].starts_with('=') && isalnum(Tmp[I][1]))
+ continue;
+ SafeConstraints.append(Tmp[I]).append({','});
+ }
+ SafeConstraints.append(Tmp.back());
+
+ return SafeConstraints;
+}
+
// Mutates indirect and inline ASM callsites iff if aggregate argument/return
// types are present with the types replaced by i32 types. The change in types
// is noted in 'spv.mutated_callsites' metadata for later restoration. For ASM
@@ -651,20 +666,18 @@ bool SPIRVPrepareFunctions::removeAggregateTypesFromCalls(Function *F) {
CB->setName("spv.named_mutated_callsite." + F->getName() + "." +
CB->getName());
- StringRef MaybeConstraints;
+ std::string Constraints;
if (auto *ASM = dyn_cast<InlineAsm>(CB->getCalledOperand())) {
- MaybeConstraints = ASM->getConstraintString();
- // We should only have one =r return for the made up ASM type.
+ Constraints = fixMultiOutputConstraintString(ASM->getConstraintString());
+
CB->setCalledOperand(InlineAsm::get(
- NewFnTy, ASM->getAsmString(),
- MaybeConstraints.substr(MaybeConstraints.find_last_of('=')),
- ASM->hasSideEffects(), ASM->isAlignStack(), ASM->getDialect(),
- ASM->canThrow()));
+ NewFnTy, ASM->getAsmString(), Constraints, ASM->hasSideEffects(),
+ ASM->isAlignStack(), ASM->getDialect(), ASM->canThrow()));
}
addFunctionTypeMutation(
F->getParent()->getOrInsertNamedMetadata("spv.mutated_callsites"),
- std::move(ChangedTypes), CB->getName(), MaybeConstraints);
+ std::move(ChangedTypes), CB->getName(), Constraints);
}
for (auto &&[CB, NewFTy] : Calls) {
>From 6df2813b766b47d39e7c93572aaae426ef324be7 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Fri, 20 Mar 2026 23:04:18 +0200
Subject: [PATCH 20/29] Handle memory constraints.
---
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 2fcb71939322a..4469e3943204c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -136,8 +136,12 @@ TargetLowering::ConstraintType
SPIRVTargetLowering::getConstraintType(StringRef Constraint) const {
// SPIR-V represents inline assembly via OpAsmINTEL where constraints are
// passed through as literals defined by client API. Return C_RegisterClass
- // for any constraint since SPIR-V does not distinguish between register,
- // immediate, or memory operands at this level.
+ // for non-memory constraints since SPIR-V does not distinguish between register,
+ // immediate, or memory operands at this level. We do have to return C_Memory
+ // for memory constraints as otherwise IRTranslator gets confused trying to
+ // allocate registers for them.
+ if (Constraint == "m")
+ return C_Memory;
return C_RegisterClass;
}
>From 06711cf56b9713aa839468f525a325414f0513c6 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Mon, 6 Apr 2026 01:11:58 +0100
Subject: [PATCH 21/29] Handle memory operands, fix broken handling of
constraint strings for multiple output cases.
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 6 ++-
llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 39 ++++++++++---------
.../Target/SPIRV/SPIRVPrepareFunctions.cpp | 8 ++--
.../SPV_INTEL_inline_assembly/inline_asm.ll | 23 ++++++++++-
4 files changed, 52 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 3857c7248ca54..df6b2d785368b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -1662,7 +1662,11 @@ Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
return &Call;
LLVMContext &Ctx = CurrF->getContext();
-
+ // TODO: this does not retain elementtype info for memory constraints, which
+ // in turn means that we lower them into pointers to i8, rather than
+ // pointers to elementtype; this can be fixed during reverse translation
+ // but we should correct it here, possibly by tweaking the function
+ // type to take TypedPointerType args.
Constant *TyC = UndefValue::get(SPIRV::getOriginalFunctionType(Call));
MDString *ConstraintString =
MDString::get(Ctx, SPIRV::getOriginalAsmConstraints(Call));
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 456a618effdc2..e31f50a7b737e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -672,8 +672,7 @@ collectInlineAsmInstrOperands(MachineInstr *MI,
if (MO.isReg() && MO.isDef()) {
if (!Ops)
return MO.getReg();
- else
- DefReg = MO.getReg();
+ DefReg = MO.getReg();
} else if (Ops) {
Ops->push_back(Idx);
}
@@ -756,22 +755,26 @@ insertInlineAsmProcess(MachineFunction &MF, SPIRVGlobalRegistry *GR,
for (unsigned IntrIdx = 3; IntrIdx < I1->getNumOperands(); ++IntrIdx)
AsmCall.addUse(I1->getOperand(IntrIdx).getReg());
- // IRTranslator gets a bit confused when lowering inline ASM with multiple,
- // outputs (which we have to spoof as single i32 return), and inserts a
- // spurious COPY & TRUNC as registers are assumed to be i64; we have to
- // clean that up here to prevent an erroneous cast on a struct to get
- // lowered into SPIR-V
- if (FTy->getReturnType()->isStructTy()) {
- MachineInstr &Copy = *++I2->getIterator();
- MachineInstr &Trunc = *++Copy.getIterator();
-
- assert(Copy.isCopy() && Trunc.getOpcode() == TargetOpcode::G_TRUNC &&
- "Unexpected successors to inline ASM with multiple outputs!");
-
- Register TruncReg = Trunc.defs().begin()->getReg();
- MRI.replaceRegWith(TruncReg, DefReg);
- invalidateAndEraseMI(GR, &Trunc);
- invalidateAndEraseMI(GR, &Copy);
+ // IRTranslator gets a bit confused when lowering inline ASM with outputs
+ // and inserts a spurious COPY & TRUNC as registers are assumed to be i64;
+ // we have to clean that up here to prevent erroneous trunc casts either on
+ // a struct (for multiple outputs) or same width integers to get lowered
+ // into SPIR-V
+ if (MRI.hasOneUse(DefReg)) {
+ MachineInstr &CopyMI = *MRI.use_instr_begin(DefReg);
+ if (CopyMI.getOpcode() == TargetOpcode::COPY) {
+ Register CopyDst = CopyMI.getOperand(0).getReg();
+ if (MRI.hasOneUse(CopyDst)) {
+ MachineInstr &TruncMI = *MRI.use_instr_begin(CopyDst);
+ if (TruncMI.getOpcode() == TargetOpcode::G_TRUNC) {
+ MRI.setType(DefReg, GR->getRegType(RetType));
+ Register TruncReg = TruncMI.defs().begin()->getReg();
+ MRI.replaceRegWith(TruncReg, DefReg);
+ invalidateAndEraseMI(GR, &TruncMI);
+ invalidateAndEraseMI(GR, &CopyMI);
+ }
+ }
+ }
}
}
for (MachineInstr *MI : ToProcess)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 813f943afb5c4..01f10e3348efc 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -604,7 +604,8 @@ static std::string fixMultiOutputConstraintString(StringRef Constraints) {
SplitString(Constraints, Tmp, ",");
std::string SafeConstraints("=r,");
for (unsigned I = 0u; I != Tmp.size() - 1; ++I) {
- if (Tmp[I].starts_with('=') && isalnum(Tmp[I][1]))
+ if (Tmp[I].starts_with('=') &&
+ (Tmp[I][1] == '&' || isalnum(Tmp[I][1])))
continue;
SafeConstraints.append(Tmp[I]).append({','});
}
@@ -668,10 +669,11 @@ bool SPIRVPrepareFunctions::removeAggregateTypesFromCalls(Function *F) {
std::string Constraints;
if (auto *ASM = dyn_cast<InlineAsm>(CB->getCalledOperand())) {
- Constraints = fixMultiOutputConstraintString(ASM->getConstraintString());
+ Constraints = ASM->getConstraintString();
CB->setCalledOperand(InlineAsm::get(
- NewFnTy, ASM->getAsmString(), Constraints, ASM->hasSideEffects(),
+ NewFnTy, ASM->getAsmString(),
+ fixMultiOutputConstraintString(Constraints), ASM->hasSideEffects(),
ASM->isAlignStack(), ASM->getDialect(), ASM->canThrow()));
}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
index ff6b6dec42ff3..30d45e54c038d 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
@@ -17,6 +17,8 @@
; CHECK-DAG: %[[#FloatTy:]] = OpTypeFloat 32
; CHECK-DAG: %[[#DoubleTy:]] = OpTypeFloat 64
; CHECK-DAG: %[[#StructTy:]] = OpTypeStruct %[[#Int32Ty]] %[[#FloatTy]] %[[#HalfTy]]
+; CHECK-DAG: %[[#StructTy1:]] = OpTypeStruct %[[#Int32Ty]] %[[#FloatTy]]
+; CHECK-DAG: %[[#Int8PtrTy:]] = OpTypePointer CrossWorkgroup %[[#Int8Ty]]
; CHECK-DAG: OpTypeFunction %[[#VoidTy]] %[[#]] %[[#]] %[[#]] %[[#Int64Ty]]
; CHECK-DAG: %[[#Fun1Ty:]] = OpTypeFunction %[[#VoidTy]]
@@ -28,6 +30,8 @@
; CHECK-DAG: %[[#Fun7Ty:]] = OpTypeFunction %[[#Int64Ty]] %[[#Int64Ty]] %[[#Int32Ty]] %[[#Int8Ty]]
; CHECK-DAG: %[[#Fun8Ty:]] = OpTypeFunction %[[#VoidTy]] %[[#Int32Ty]] %[[#DoubleTy]]
; CHECK-DAG: %[[#Fun9Ty:]] = OpTypeFunction %[[#StructTy]] %[[#Int32Ty]] %[[#FloatTy]] %[[#HalfTy]]
+; CHECK-DAG: %[[#Fun10Ty:]] = OpTypeFunction %[[#Int32Ty]] %[[#Int8PtrTy]] %[[#Int32Ty]] %[[#Int8PtrTy]]
+; CHECK-DAG: %[[#Fun11Ty:]] = OpTypeFunction %[[#StructTy1]] %[[#Int8PtrTy]] %[[#Int32Ty]] %[[#FloatTy]]
; CHECK-DAG: %[[#Const2:]] = OpConstant %[[#FloatTy]] 2
; CHECK-DAG: %[[#Const123:]] = OpConstant %[[#Int32Ty]] 123
@@ -47,7 +51,9 @@
; CHECK-DAG: %[[#Asm9:]] = OpAsmINTEL %[[#Int64Ty]] %[[#Fun7Ty]] %[[#Dialect]] "icmdext $0 $3 $1 $2" "=r,r,r,r"
; CHECK-DAG: %[[#Asm10:]] = OpAsmINTEL %[[#VoidTy]] %[[#Fun8Ty]] %[[#Dialect]] "constcmd $0 $1" "r,r"
; CHECK-DAG: %[[#Asm11:]] = OpAsmINTEL %[[#VoidTy]] %[[#Fun8Ty]] %[[#Dialect]] "constcmd $0 $1" "i,i"
-; CHECK-DAG: %[[#Asm12:]] = OpAsmINTEL %[[#StructTy]] %[[#Fun9Ty]] %[[#Dialect]] "cmdext $0 $4 $5\n cmdext $2 $5 $6\n cmdext $3 $4 $6" "=&r,=&r,=&r,r,r,r"
+; CHECK-DAG: %[[#Asm12:]] = OpAsmINTEL %[[#StructTy]] %[[#Fun9Ty]] %[[#Dialect]] "cmdext $0 $4 $5\n cmdext $1 $5 $6\n cmdext $2 $4 $6" "=&r,=&r,=&r,r,r,r"
+; CHECK-DAG: %[[#Asm13:]] = OpAsmINTEL %[[#Int32Ty]] %[[#Fun10Ty]] %[[#Dialect]] "icmdext $0 $2 $3"
+; CHECK-DAG: %[[#Asm14:]] = OpAsmINTEL %[[#StructTy1]] %[[#Fun11Ty]] %[[#Dialect]] "cmdext $0 $3 $4\n cmdext $1 $3 $4\n $2 $3 $4" "=r,=&r,=*m, r, r"
; CHECK-NO: OpAsmINTEL
; CHECK: OpFunction
@@ -66,6 +72,10 @@
; CHECK-NEXT: OpCompositeExtract %[[#Int32Ty]] %[[#StructRet]] 0
; CHECK-NEXT: OpCompositeExtract %[[#FloatTy]] %[[#StructRet]] 1
; CHECK-NEXT: OpCompositeExtract %[[#HalfTy]] %[[#StructRet]] 2
+; CHECK: OpAsmCallINTEL %[[#Int32Ty]] %[[#Asm13]]
+; CHECK: %[[#StructRet1:]] = OpAsmCallINTEL %[[#StructTy1]] %[[#Asm14]]
+; CHECK-NEXT: OpCompositeExtract %[[#Int32Ty]] %[[#StructRet1]] 0
+; CHECK-NEXT: OpCompositeExtract %[[#FloatTy]] %[[#StructRet1]] 1
; CHECK-NO: OpAsmCallINTEL
define spir_kernel void @foo(ptr addrspace(1) %_arg_int, ptr addrspace(1) %_arg_float, ptr addrspace(1) %_arg_half, i64 %_lng) {
@@ -97,12 +107,21 @@ define spir_kernel void @foo(ptr addrspace(1) %_arg_int, ptr addrspace(1) %_arg_
call void asm "constcmd $0 $1", "r,r"(i32 123, double 42.0)
call void asm "constcmd $0 $1", "i,i"(i32 123, double 42.0)
; inline asm: multiple outputs, hence aggregate return
- %res_struct = call { i32, float, half } asm sideeffect "cmdext $0 $4 $5\0A cmdext $2 $5 $6\0A cmdext $3 $4 $6", "=&r,=&r,=&r,r,r,r"(i32 %i1, float %f1, half %h1)
+ %res_struct = call { i32, float, half } asm sideeffect "cmdext $0 $4 $5\0A cmdext $1 $5 $6\0A cmdext $2 $4 $6", "=&r,=&r,=&r,r,r,r"(i32 %i1, float %f1, half %h1)
%asmresult = extractvalue { i32, float, half } %res_struct, 0
%asmresult3 = extractvalue { i32, float, half } %res_struct, 1
%asmresult4 = extractvalue { i32, float, half } %res_struct, 2
store i32 %asmresult, ptr addrspace(1) %_arg_int
store float %asmresult3, ptr addrspace(1) %_arg_float
store half %asmresult4, ptr addrspace(1) %_arg_half
+ ; inline asm: two outputs but one is a mem operand
+ %asmtmp = call i32 asm sideeffect "icmdext $0 $2 $3", "=&r,=*m,r,*m,~{memory}"(ptr addrspace(1) elementtype(i32) %_arg_int, i32 %i1, ptr addrspace(1) elementtype(float) %_arg_float)
+ store i32 %asmtmp, ptr addrspace(1) %_arg_int
+ ; inline asm: multiple outputs out of which one is a mem operand
+ %res_struct1 = call { i32, float } asm sideeffect "cmdext $0 $3 $4\0A cmdext $1 $3 $4\0A $2 $3 $4", "=r,=&r,=*m, r, r"(ptr addrspace(1) elementtype(i32) %_arg_int, i32 %i1, float %f1)
+ %asmresult1 = extractvalue { i32, float } %res_struct1, 0
+ %asmresult2 = extractvalue { i32, float } %res_struct1, 1
+ store i32 %asmresult1, ptr addrspace(1) %_arg_int
+ store float %asmresult2, ptr addrspace(1) %_arg_float
ret void
}
>From 3b07cb1c8680c75a69cc86f57200c2a7ea76799f Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Mon, 6 Apr 2026 01:22:15 +0100
Subject: [PATCH 22/29] Fix formatting.
---
llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 01f10e3348efc..a6572ae3c0f20 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -604,8 +604,7 @@ static std::string fixMultiOutputConstraintString(StringRef Constraints) {
SplitString(Constraints, Tmp, ",");
std::string SafeConstraints("=r,");
for (unsigned I = 0u; I != Tmp.size() - 1; ++I) {
- if (Tmp[I].starts_with('=') &&
- (Tmp[I][1] == '&' || isalnum(Tmp[I][1])))
+ if (Tmp[I].starts_with('=') && (Tmp[I][1] == '&' || isalnum(Tmp[I][1])))
continue;
SafeConstraints.append(Tmp[I]).append({','});
}
>From 7d1135d9ae822297d45d4f3aef908d239c53b415 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Mon, 6 Apr 2026 01:35:48 +0100
Subject: [PATCH 23/29] Fix formatting.
---
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 4469e3943204c..6c3363259b901 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -136,10 +136,10 @@ TargetLowering::ConstraintType
SPIRVTargetLowering::getConstraintType(StringRef Constraint) const {
// SPIR-V represents inline assembly via OpAsmINTEL where constraints are
// passed through as literals defined by client API. Return C_RegisterClass
- // for non-memory constraints since SPIR-V does not distinguish between register,
- // immediate, or memory operands at this level. We do have to return C_Memory
- // for memory constraints as otherwise IRTranslator gets confused trying to
- // allocate registers for them.
+ // for non-memory constraints since SPIR-V does not distinguish between
+ // register, immediate, or memory operands at this level. We do have to return
+ // C_Memory for memory constraints as otherwise IRTranslator gets confused
+ // trying to allocate registers for them.
if (Constraint == "m")
return C_Memory;
return C_RegisterClass;
>From a904ff58ebd33c065f60c6944967cd2d82384429 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Wed, 22 Apr 2026 21:16:14 +0100
Subject: [PATCH 24/29] Update test.
---
.../SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
index 30d45e54c038d..3f02e5a790146 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_inline_assembly/inline_asm.ll
@@ -7,7 +7,7 @@
; CHECK: OpCapability AsmINTEL
; CHECK: OpExtension "SPV_INTEL_inline_assembly"
-; CHECK-COUNT-8: OpDecorate %[[#]] SideEffectsINTEL
+; CHECK-COUNT-11: OpDecorate %[[#]] SideEffectsINTEL
; CHECK-DAG: %[[#VoidTy:]] = OpTypeVoid
; CHECK-DAG: %[[#Int8Ty:]] = OpTypeInt 8 0
>From 9aefb32955c475b514bcb398824f773f911beb85 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Wed, 22 Apr 2026 21:19:27 +0100
Subject: [PATCH 25/29] Fix typo.
---
llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index a6572ae3c0f20..e49b1681417ae 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -613,10 +613,10 @@ static std::string fixMultiOutputConstraintString(StringRef Constraints) {
return SafeConstraints;
}
-// Mutates indirect and inline ASM callsites iff if aggregate argument/return
-// types are present with the types replaced by i32 types. The change in types
-// is noted in 'spv.mutated_callsites' metadata for later restoration. For ASM
-// we also have to mutate the constraint string as IRTranslator tries to handle
+// Mutates indirect and inline ASM callsites iff aggregate argument/return types
+// are present with the types replaced by i32 types. The change in types is
+// noted in 'spv.mutated_callsites' metadata for later restoration. For ASM we
+// also have to mutate the constraint string as IRTranslator tries to handle
// multiple outputs and expects an aggregate return type in their presence.
bool SPIRVPrepareFunctions::removeAggregateTypesFromCalls(Function *F) {
if (F->isDeclaration() || F->isIntrinsic())
>From 622d6ba5b07d6222e91ecdedb7309a4132bcaa28 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Tue, 19 May 2026 03:04:00 +0300
Subject: [PATCH 26/29] We should not disable untyped ptrs when using the
translator to generate SPIR-V.
---
clang/lib/Driver/ToolChains/HIPAMD.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/lib/Driver/ToolChains/HIPAMD.cpp b/clang/lib/Driver/ToolChains/HIPAMD.cpp
index 7e38ecbb9b9f0..70c109be6943f 100644
--- a/clang/lib/Driver/ToolChains/HIPAMD.cpp
+++ b/clang/lib/Driver/ToolChains/HIPAMD.cpp
@@ -188,7 +188,7 @@ void AMDGCN::Linker::constructLinkAndEmitSpirvCommand(
// Emit SPIR-V binary using the translator
llvm::opt::ArgStringList TrArgs{
"--spirv-max-version=1.6",
- "--spirv-ext=+all,-SPV_KHR_untyped_pointers",
+ "--spirv-ext=+all",
"--spirv-allow-unknown-intrinsics",
"--spirv-lower-const-expr",
"--spirv-preserve-auxdata",
>From 9d8f60d613a8d2df25a1b3882a57625136c64ca6 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Tue, 19 May 2026 03:49:12 +0300
Subject: [PATCH 27/29] Fix tests.
---
clang/test/Driver/hip-toolchain-no-rdc.hip | 2 +-
clang/test/Driver/spirv-amd-toolchain.c | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/clang/test/Driver/hip-toolchain-no-rdc.hip b/clang/test/Driver/hip-toolchain-no-rdc.hip
index 975515bd0cb40..f4cd703547ac0 100644
--- a/clang/test/Driver/hip-toolchain-no-rdc.hip
+++ b/clang/test/Driver/hip-toolchain-no-rdc.hip
@@ -214,7 +214,7 @@
// AMDGCNSPIRV: "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}}"-emit-llvm-bc" {{.*}}"-fembed-bitcode=marker" "-disable-llvm-passes" {{.*}} "-o" "[[AMDGCNSPV_BC:.*bc]]"
// AMDGCNSPIRV: {{".*llvm-link.*"}} "-o" "[[AMDGCNSPV_TMP:.*bc]]" "[[AMDGCNSPV_BC]]"
-// AMDGCNSPIRV: {{".*llvm-spirv.*"}} "--spirv-max-version=1.6" "--spirv-ext=+all,-SPV_KHR_untyped_pointers" {{.*}} "[[AMDGCNSPV_TMP]]" {{.*}}"-o" "[[AMDGCNSPV_CO:.*out]]"
+// AMDGCNSPIRV: {{".*llvm-spirv.*"}} "--spirv-max-version=1.6" "--spirv-ext=+all" {{.*}} "[[AMDGCNSPV_TMP]]" {{.*}}"-o" "[[AMDGCNSPV_CO:.*out]]"
// AMDGCNSPIRV: "-cc1" "-triple" "amdgcn-amd-amdhsa" {{.*}}"-emit-obj" {{.*}}"-target-cpu" "gfx900"{{.*}} "-o" "[[GFX900_OBJ:.*o]]"
// AMDGCNSPIRV: {{".*lld.*"}} {{.*}}"-plugin-opt=mcpu=gfx900" {{.*}} "-o" "[[GFX900_CO:.*out]]" {{.*}}"[[GFX900_OBJ]]"
// AMDGCNSPIRV: {{".*clang-offload-bundler.*"}} "-type=o"
diff --git a/clang/test/Driver/spirv-amd-toolchain.c b/clang/test/Driver/spirv-amd-toolchain.c
index 4c7a673ef85fe..c9bba1e437e11 100644
--- a/clang/test/Driver/spirv-amd-toolchain.c
+++ b/clang/test/Driver/spirv-amd-toolchain.c
@@ -21,7 +21,7 @@
// RUN: | FileCheck %s --check-prefix=INVOCATION
// INVOCATION: "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}}"-disable-llvm-optzns" {{.*}} "-o" "[[OUTPUT:.+]]" "-x" "c"
// INVOCATION: "{{.*}}llvm-link" "-o" "[[LINKED_OUTPUT:.+]]" "[[OUTPUT]]"
-// INVOCATION: "{{.*}}llvm-spirv" "--spirv-max-version=1.6" "--spirv-ext=+all,-SPV_KHR_untyped_pointers" "--spirv-allow-unknown-intrinsics" "--spirv-lower-const-expr" "--spirv-preserve-auxdata" "--spirv-debug-info-version=nonsemantic-shader-200" "[[LINKED_OUTPUT]]" "-o" "a.out"
+// INVOCATION: "{{.*}}llvm-spirv" "--spirv-max-version=1.6" "--spirv-ext=+all" "--spirv-allow-unknown-intrinsics" "--spirv-lower-const-expr" "--spirv-preserve-auxdata" "--spirv-debug-info-version=nonsemantic-shader-200" "[[LINKED_OUTPUT]]" "-o" "a.out"
// RUN: %clang -### -use-spirv-backend --target=spirv64-amd-amdhsa %s 2>&1 \
// RUN: | FileCheck %s --check-prefix=INVOCATION-SPIRV-BACKEND
>From 5837f684723cb458667980f7dcc5d79d8536f354 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Sat, 23 May 2026 23:32:49 +0300
Subject: [PATCH 28/29] Add support for buffer specific ASes, and assorted
fixes.
---
clang/lib/CodeGen/CodeGenModule.cpp | 5 +-
clang/test/CodeGen/target-data.c | 2 +-
llvm/include/llvm/IR/IntrinsicsSPIRV.td | 5 +
llvm/lib/IR/Type.cpp | 4 +
llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 16 +-
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 11 +-
llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 26 +-
llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 4 +-
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 37 +++
llvm/lib/Target/SPIRV/SPIRVISelLowering.h | 3 +
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 33 +-
.../SPIRV/SPIRVLegalizeZeroSizeArrays.cpp | 3 +-
llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp | 57 ++--
llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.h | 3 +-
llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 5 +-
.../Target/SPIRV/SPIRVPrepareFunctions.cpp | 77 ++++-
.../Target/SPIRV/SPIRVPushConstantAccess.cpp | 3 +-
llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp | 9 +-
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 8 +-
llvm/lib/Target/SPIRV/SPIRVUtils.h | 7 +-
llvm/lib/TargetParser/TargetDataLayout.cpp | 11 +-
...cnspirv-lower-buffer-fat-pointers-calls.ll | 81 +++++
...irv-lower-buffer-fat-pointers-constants.ll | 305 ++++++++++++++++++
...wer-buffer-fat-pointers-dead-intrinsics.ll | 9 +
...nspirv-lower-buffer-fat-pointers-memops.ll | 162 ++++++++++
...-lower-buffer-fat-pointers-p7-in-memory.ll | 75 +++++
...fer-fat-pointers-unoptimized-debug-data.ll | 142 ++++++++
27 files changed, 1036 insertions(+), 67 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-calls.ll
create mode 100644 llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-constants.ll
create mode 100644 llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-dead-intrinsics.ll
create mode 100644 llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-memops.ll
create mode 100644 llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-p7-in-memory.ll
create mode 100644 llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-unoptimized-debug-data.ll
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 236738e9975d3..3afeb5e4cf7c5 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -415,7 +415,10 @@ static void checkDataLayoutConsistency(const TargetInfo &Target,
if (Target.hasIbm128Type())
Check("__ibm128", llvm::Type::getPPC_FP128Ty(Context), Target.Ibm128Align);
- Check("void*", llvm::PointerType::getUnqual(Context), Target.PointerAlign);
+ Check("void*",
+ llvm::PointerType::get(Context,
+ Target.getTargetAddressSpace(LangAS::Default)),
+ Target.PointerAlign);
if (Target.vectorsAreElementAligned() != DL.vectorsAreElementAligned()) {
llvm::errs() << "Datalayout for target " << Triple.str()
diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c
index 0047e1377a7f8..a73531a7714d6 100644
--- a/clang/test/CodeGen/target-data.c
+++ b/clang/test/CodeGen/target-data.c
@@ -259,7 +259,7 @@
// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -o - -emit-llvm %s | \
// RUN: FileCheck %s -check-prefix=AMDGPUSPIRV64
-// AMDGPUSPIRV64: target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n32:64-S32-G1-P4-A0"
+// AMDGPUSPIRV64: target datalayout = "e-m:e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A0-G1-P4-ni:7:8:9"
// RUN: %clang_cc1 -triple spirv-unknown-vulkan -o - -emit-llvm %s | \
// RUN: FileCheck %s -check-prefix=SPIRVVULKAN
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 26996774afa87..d728ed080dca4 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -354,6 +354,11 @@ def int_spv_rsqrt : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty]
def int_spv_generic_cast_to_ptr_explicit
: DefaultAttrsIntrinsic<[llvm_anyptr_ty], [generic_ptr_ty],
[IntrNoMem, NoUndef<RetIndex>]>;
+ // Encode SPIR-V invalid but potentially target valid casts via intrinsic that
+ // always gets lowered to a function.
+ def int_spv_opaque_ptr_cast
+ : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [llvm_anyptr_ty],
+ [IntrNoMem, NoUndef<RetIndex>]>;
def int_spv_unpackhalf2x16 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_i32_ty], [IntrNoMem]>;
def int_spv_packhalf2x16 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index 498e78d4b0cc8..4be20db1c0d33 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -1082,6 +1082,10 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) {
return TargetTypeInfo(
ArrayType::get(Type::getInt8Ty(C), Ty->getIntParameter(0)),
TargetExtType::CanBeGlobal);
+ if (Name == "spirv.$TypedPointerType")
+ return TargetTypeInfo(PointerType::get(C, 4), TargetExtType::HasZeroInit,
+ TargetExtType::CanBeGlobal, TargetExtType::CanBeLocal,
+ TargetExtType::CanBeVectorElement);
if (Name.starts_with("spirv."))
return TargetTypeInfo(PointerType::get(C, 0), TargetExtType::HasZeroInit,
TargetExtType::CanBeGlobal,
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 1febfc21ed3b0..6ef8715783748 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -510,10 +510,10 @@ static Register buildBuiltinVariableLoad(
SPIRV::LinkageType::Import}) {
Register NewRegister =
MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::pIDRegClass);
- MIRBuilder.getMRI()->setType(
- NewRegister,
- LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function),
- GR->getPointerSize()));
+ unsigned AS = storageClassToAddressSpace(
+ SPIRV::StorageClass::Function, GR->CurMF->getTarget().getTargetTriple());
+ MIRBuilder.getMRI()->setType(NewRegister,
+ LLT::pointer(AS, GR->getPointerSize(AS)));
SPIRVTypeInst PtrType = GR->getOrCreateSPIRVPointerType(
VariableType, MIRBuilder, SPIRV::StorageClass::Input);
GR->assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
@@ -2929,7 +2929,8 @@ static SPIRVTypeInst
getOrCreateSPIRVDeviceEventPointer(MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
LLVMContext &Context = MIRBuilder.getMF().getFunction().getContext();
- unsigned SC1 = storageClassToAddressSpace(SPIRV::StorageClass::Generic);
+ unsigned SC1 = storageClassToAddressSpace(
+ SPIRV::StorageClass::Generic, GR->CurMF->getTarget().getTargetTriple());
Type *PtrType = PointerType::get(Context, SC1);
return GR->getOrCreateSPIRVType(PtrType, MIRBuilder,
SPIRV::AccessQualifier::ReadWrite, true);
@@ -2960,8 +2961,9 @@ static bool buildEnqueueKernel(const SPIRV::IncomingCall *Call,
assert(LocalSizeTy && "Local size type is expected");
const uint64_t LocalSizeNum =
cast<ArrayType>(LocalSizeTy)->getNumElements();
- unsigned SC = storageClassToAddressSpace(SPIRV::StorageClass::Generic);
- const LLT LLType = LLT::pointer(SC, GR->getPointerSize());
+ unsigned SC = storageClassToAddressSpace(
+ SPIRV::StorageClass::Generic, GR->CurMF->getTarget().getTargetTriple());
+ const LLT LLType = LLT::pointer(SC, GR->getPointerSize(SC));
const SPIRVTypeInst PointerSizeTy = GR->getOrCreateSPIRVPointerType(
Int32Ty, MIRBuilder, SPIRV::StorageClass::Function);
for (unsigned I = 0; I < LocalSizeNum; ++I) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index b6e71c7b76348..48f12faf8db6f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -1638,12 +1638,17 @@ void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) {
// pointer.
void SPIRVEmitIntrinsics::simplifyNullAddrSpaceCasts() {
for (Instruction &I : make_early_inc_range(instructions(CurrF)))
- if (auto *ASC = dyn_cast<AddrSpaceCastInst>(&I))
+ if (auto *ASC = dyn_cast<AddrSpaceCastInst>(&I)) {
if (isa<ConstantPointerNull>(ASC->getPointerOperand())) {
- ASC->replaceAllUsesWith(
- ConstantPointerNull::get(cast<PointerType>(ASC->getType())));
+ if (ASC->getType()->isVectorTy()) {
+ ASC->replaceAllUsesWith(ConstantVector::getNullValue(ASC->getType()));
+ } else {
+ ASC->replaceAllUsesWith(
+ ConstantPointerNull::get(cast<PointerType>(ASC->getType())));
+ }
ASC->eraseFromParent();
}
+ }
}
void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 343311fb44475..06ab77566753c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -763,7 +763,7 @@ SPIRVGlobalRegistry::getOrCreateConstNullPtr(MachineIRBuilder &MIRBuilder,
if (Res.isValid())
return Res;
- LLT LLTy = LLT::pointer(AddressSpace, getPointerSize());
+ LLT LLTy = LLT::pointer(AddressSpace, getPointerSize(AddressSpace));
Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);
CurMF->getRegInfo().setRegClass(Res, &SPIRV::pIDRegClass);
assignSPIRVTypeToVReg(SpvType, Res, *CurMF);
@@ -863,8 +863,8 @@ Register SPIRVGlobalRegistry::buildGlobalVariable(
// Set to Reg the same type as ResVReg has.
auto MRI = MIRBuilder.getMRI();
if (Reg != ResVReg) {
- LLT RegLLTy =
- LLT::pointer(MRI->getType(ResVReg).getAddressSpace(), getPointerSize());
+ unsigned AS = MRI->getType(ResVReg).getAddressSpace();
+ LLT RegLLTy = LLT::pointer(AS, getPointerSize(AS));
MRI->setType(Reg, RegLLTy);
assignSPIRVTypeToVReg(BaseType, Reg, MIRBuilder.getMF());
} else {
@@ -2033,7 +2033,8 @@ SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSPIRVPointerTypeInternal(
SPIRVTypeInst BaseType, MachineIRBuilder &MIRBuilder,
SPIRV::StorageClass::StorageClass SC) {
const Type *PointerElementType = getTypeForSPIRVType(BaseType);
- unsigned AddressSpace = storageClassToAddressSpace(SC);
+ unsigned AddressSpace =
+ storageClassToAddressSpace(SC, CurMF->getTarget().getTargetTriple());
if (const MachineInstr *MI = findMI(PointerElementType, AddressSpace, CurMF))
return MI;
Type *Ty = TypedPointerType::get(const_cast<Type *>(PointerElementType),
@@ -2105,28 +2106,33 @@ SPIRVGlobalRegistry::getRegClass(SPIRVTypeInst SpvType) const {
return &SPIRV::iIDRegClass;
}
-inline unsigned getAS(SPIRVTypeInst SpvType) {
+inline unsigned getAS(SPIRVTypeInst SpvType, const Triple &TT) {
return storageClassToAddressSpace(
static_cast<SPIRV::StorageClass::StorageClass>(
- SpvType->getOperand(1).getImm()));
+ SpvType->getOperand(1).getImm()), TT);
}
LLT SPIRVGlobalRegistry::getRegType(SPIRVTypeInst SpvType) const {
unsigned Opcode = SpvType ? SpvType->getOpcode() : 0;
+ const Triple &TT = CurMF->getTarget().getTargetTriple();
switch (Opcode) {
case SPIRV::OpTypeInt:
case SPIRV::OpTypeFloat:
case SPIRV::OpTypeBool:
return LLT::scalar(getScalarOrVectorBitWidth(SpvType));
- case SPIRV::OpTypePointer:
- return LLT::pointer(getAS(SpvType), getPointerSize());
+ case SPIRV::OpTypePointer: {
+ unsigned AS = getAS(SpvType, TT);
+ return LLT::pointer(AS, getPointerSize(AS));
+ }
case SPIRV::OpTypeVector: {
SPIRVTypeInst ElemType = getScalarOrVectorComponentType(SpvType);
LLT ET;
switch (ElemType ? ElemType->getOpcode() : 0) {
- case SPIRV::OpTypePointer:
- ET = LLT::pointer(getAS(ElemType), getPointerSize());
+ case SPIRV::OpTypePointer: {
+ unsigned AS = getAS(ElemType, TT);
+ ET = LLT::pointer(AS, getPointerSize(AS));
break;
+ }
case SPIRV::OpTypeInt:
case SPIRV::OpTypeFloat:
case SPIRV::OpTypeBool:
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index f05bdc2cc9861..427aab376d69d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -418,8 +418,8 @@ class SPIRVGlobalRegistry : public SPIRVIRMapping {
getPointerStorageClass(SPIRVTypeInst Type) const;
// Return the number of bits SPIR-V pointers and size_t variables require.
- unsigned getPointerSize() const {
- return DL.getPointerSizeInBits(/* AS = */ 0);
+ unsigned getPointerSize(unsigned AS = 0) const {
+ return DL.getPointerSizeInBits(AS);
}
// Returns true if two types are defined and are compatible in a sense of
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index eb16c9a314a23..c5499e9d6c1ea 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -21,6 +21,7 @@
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsSPIRV.h"
+#include "llvm/Support/AMDGPUAddrSpace.h"
#define DEBUG_TYPE "spirv-lower"
@@ -102,6 +103,12 @@ MVT SPIRVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
else if (VT.getVectorElementType() == MVT::i8)
return MVT::v4i8;
}
+ // if (STI.getTargetTriple().getVendor() == Triple::VendorType::AMD) {
+ // if (VT.isVector() &&
+ // VT.getVectorElementType().isExtended())
+ // return MVT::v8i32;
+ // }
+
return getRegisterType(Context, VT);
}
@@ -684,3 +691,33 @@ SPIRVTargetLowering::shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const {
// SPIR-V only supports atomic exchange for integer and floating-point types.
return AtomicExpansionKind::None;
}
+
+// These are specific to AMDGCN flavoured SPIR-V, and mirror the AMDGPU
+// implementation; they are required because buffer operations and the machinery
+// around them are somewhat special.
+MVT SPIRVTargetLowering::getPointerTy(const DataLayout &DL, unsigned AS) const {
+ if (STI.getTargetTriple().getVendor() != Triple::VendorType::AMD)
+ return TargetLowering::getPointerTy(DL, AS);
+
+ if (AMDGPUAS::BUFFER_FAT_POINTER == AS && DL.getPointerSizeInBits(AS) == 160)
+ return MVT::amdgpuBufferFatPointer;
+ if (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
+ DL.getPointerSizeInBits(AS) == 192)
+ return MVT::amdgpuBufferStridedPointer;
+
+ return TargetLowering::getPointerTy(DL, AS);
+}
+
+MVT SPIRVTargetLowering::getPointerMemTy(const DataLayout &DL,
+ unsigned AS) const {
+ if (STI.getTargetTriple().getVendor() != Triple::VendorType::AMD)
+ return TargetLowering::getPointerMemTy(DL, AS);
+
+ if ((AMDGPUAS::BUFFER_FAT_POINTER == AS &&
+ DL.getPointerSizeInBits(AS) == 160) ||
+ (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
+ DL.getPointerSizeInBits(AS) == 192))
+ return MVT::v8i32;
+
+ return TargetLowering::getPointerMemTy(DL, AS);
+}
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
index 6af0bbcc9818d..60710610c6c28 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
@@ -82,6 +82,9 @@ class SPIRVTargetLowering : public TargetLowering {
bool shouldIssueAtomicLoadForAtomicEmulationLoop() const override {
return false;
}
+
+ MVT getPointerTy(const DataLayout &DL, unsigned AS) const override;
+ MVT getPointerMemTy(const DataLayout &DL, unsigned AS) const override;
};
} // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 02b41b9cbd64c..df03b0cf64bf8 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -2565,9 +2565,9 @@ SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
SPIRVTypeInst GenericPtrTy =
GR.changePointerStorageClass(SrcPtrTy, SPIRV::StorageClass::Generic, I);
Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass);
- MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace(
- SPIRV::StorageClass::Generic),
- GR.getPointerSize()));
+ unsigned AS = storageClassToAddressSpace(SPIRV::StorageClass::Generic,
+ STI.getTargetTriple());
+ MRI->setType(Tmp, LLT::pointer(AS, GR.getPointerSize(AS)));
MachineFunction *MF = I.getParent()->getParent();
GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF);
MachineInstrBuilder MIB = buildSpecConstantOp(
@@ -2626,6 +2626,25 @@ bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
I, ResVReg, MIB->getOperand(0).getReg(), getUcharPtrTypeReg(I, DstSC),
static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr))
.constrainAllUses(TII, TRI, RBI);
+ } else if (STI.getTargetTriple().getVendor() == Triple::VendorType::AMD &&
+ isUSMStorageClass(SrcSC) && isUSMStorageClass(DstSC)) {
+ // For AMDGCN flavoured SPIR-V we repurpose DeviceOnlyINTEL and
+ // HostOnlyINTEL (which we do not use) to represent Buffer specific
+ // storage. Pointers to Buffer are essentially opaque handles, and have
+ // non trivial lowering, so we only care about retaining information about
+ // in their nature in SPIR-V, as they can only be handled properly during
+ // finalisation. Since they are inter-castable, and there is no valid
+ // SPIR-V cast between DeviceOnlyINTEL and HostOnlyIntel, we use a PtrToU
+ // followed by an UToPtr pattern to encode the cast, pattern that we match
+ // and retrieve at finalisation.
+ auto I64Ty = GR.getOrCreateSPIRVIntegerType(64, I, TII);
+ Register Tmp = createVirtualRegister(I64Ty, &GR, MRI, MRI->getMF());
+ auto MIB = buildSpecConstantOp(I, Tmp, SrcPtr, GR.getSPIRVTypeID(I64Ty),
+ SPIRV::Opcode::ConvertPtrToU);
+ MIB.constrainAllUses(TII, TRI, RBI);
+ buildSpecConstantOp(I, ResVReg, Tmp, getUcharPtrTypeReg(I, DstSC),
+ SPIRV::Opcode::ConvertUToPtr)
+ .constrainAllUses(TII, TRI, RBI);
}
return true;
}
@@ -6809,7 +6828,8 @@ bool SPIRVInstructionSelector::selectModf(Register ResVReg,
MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
MIRBuilder.getMRI()->setType(
PtrTyReg,
- LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function),
+ LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function,
+ STI.getTargetTriple()),
GR.getPointerSize()));
// Assign SPIR-V type of the pointer type of the alloca variable to the
@@ -6924,10 +6944,11 @@ bool SPIRVInstructionSelector::loadBuiltinInputID(
// Create new register for the input ID builtin variable.
Register NewRegister =
MIRBuilder.getMRI()->createVirtualRegister(GR.getRegClass(PtrType));
+ unsigned AS = storageClassToAddressSpace(SPIRV::StorageClass::Input,
+ STI.getTargetTriple());
MIRBuilder.getMRI()->setType(
NewRegister,
- LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Input),
- GR.getPointerSize()));
+ LLT::pointer(AS, GR.getPointerSize(AS)));
GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
// Build global variable with the necessary decorations for the input ID
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizeZeroSizeArrays.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizeZeroSizeArrays.cpp
index 09697b59c374a..0991ed14e7df0 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizeZeroSizeArrays.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizeZeroSizeArrays.cpp
@@ -119,7 +119,8 @@ Type *SPIRVLegalizeZeroSizeArraysImpl::legalizeType(Type *Ty) {
if (isa<ArrayType>(Ty)) {
LegalizedTy = PointerType::get(
Ty->getContext(),
- storageClassToAddressSpace(SPIRV::StorageClass::Generic));
+ storageClassToAddressSpace(SPIRV::StorageClass::Generic,
+ TM.getTargetTriple()));
} else if (StructType *StructTy = dyn_cast<StructType>(Ty)) {
SmallVector<Type *, 8> ElemTypes;
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index dd48aa3ea2f0f..bfd937d47b6f1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -14,6 +14,7 @@
#include "SPIRV.h"
#include "SPIRVGlobalRegistry.h"
#include "SPIRVSubtarget.h"
+#include "SPIRVTargetMachine.h"
#include "SPIRVUtils.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
@@ -38,7 +39,8 @@ LegalityPredicate typeOfExtendedScalars(unsigned TypeIdx, bool IsExtendedInts) {
};
}
-SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
+SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST,
+ const TargetMachine &TM) {
using namespace TargetOpcode;
this->ST = &ST;
@@ -81,23 +83,24 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
const LLT v2s8 = LLT::fixed_vector(2, 8);
const LLT v2s1 = LLT::fixed_vector(2, 1);
- const unsigned PSize = ST.getPointerSize();
- const LLT p0 = LLT::pointer(0, PSize); // Function
- const LLT p1 = LLT::pointer(1, PSize); // CrossWorkgroup
- const LLT p2 = LLT::pointer(2, PSize); // UniformConstant
- const LLT p3 = LLT::pointer(3, PSize); // Workgroup
- const LLT p4 = LLT::pointer(4, PSize); // Generic
- const LLT p5 =
- LLT::pointer(5, PSize); // Input, SPV_INTEL_usm_storage_classes (Device)
- const LLT p6 = LLT::pointer(6, PSize); // SPV_INTEL_usm_storage_classes (Host)
- const LLT p7 = LLT::pointer(7, PSize); // Input
- const LLT p8 = LLT::pointer(8, PSize); // Output
- const LLT p9 =
- LLT::pointer(9, PSize); // CodeSectionINTEL, SPV_INTEL_function_pointers
- const LLT p10 = LLT::pointer(10, PSize); // Private
- const LLT p11 = LLT::pointer(11, PSize); // StorageBuffer
- const LLT p12 = LLT::pointer(12, PSize); // Uniform
- const LLT p13 = LLT::pointer(13, PSize); // PushConstant
+ const Triple &TT = TM.getTargetTriple();
+ const LLT p0 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function, TT), TM.getPointerSizeInBits(0)); // Function
+ const LLT p1 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::CrossWorkgroup, TT), TM.getPointerSizeInBits(1)); // CrossWorkgroup
+ const LLT p2 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::UniformConstant, TT), TM.getPointerSizeInBits(2)); // UniformConstant
+ const LLT p3 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Workgroup, TT), TM.getPointerSizeInBits(3)); // Workgroup
+ const LLT p4 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Generic, TT), TM.getPointerSizeInBits(4)); // Generic
+ // Input, SPV_INTEL_usm_storage_classes (Device)
+ const LLT p5 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::DeviceOnlyINTEL, TT), TM.getPointerSizeInBits(5));
+ // SPV_INTEL_usm_storage_classes (Host)
+ const LLT p6 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::HostOnlyINTEL, TT), TM.getPointerSizeInBits(6));
+ const LLT p7 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Input, TT), TM.getPointerSizeInBits(7)); // Input
+ const LLT p8 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Output, TT), TM.getPointerSizeInBits(8)); // Output
+ // CodeSectionINTEL, SPV_INTEL_function_pointers
+ const LLT p9 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::CodeSectionINTEL, TT), TM.getPointerSizeInBits(9));
+ const LLT p10 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Private, TT), TM.getPointerSizeInBits(10)); // Private
+ const LLT p11 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::StorageBuffer, TT), TM.getPointerSizeInBits(11)); // StorageBuffer
+ const LLT p12 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Uniform, TT), TM.getPointerSizeInBits(12)); // Uniform
+ const LLT p13 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::PushConstant, TT), TM.getPointerSizeInBits(13)); // PushConstant
// TODO: remove copy-pasting here by using concatenation in some way.
auto allPtrsScalarsAndVectors = {
@@ -385,6 +388,15 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
return DstTy.isPointerVector() && SrcTy.isVector() &&
!SrcTy.isPointer() &&
DstTy.getNumElements() == SrcTy.getNumElements();
+ })
+ .legalIf(sameSize(0, 1))
+ .widenScalarIf(smallerThan(1, 0),
+ [](const LegalityQuery &Query) {
+ return std::pair(
+ 1, LLT::scalar(Query.Types[0].getSizeInBits()));
+ })
+ .narrowScalarIf(largerThan(1, 0), [](const LegalityQuery &Query) {
+ return std::pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
});
getActionDefinitionsBuilder(G_PTRTOINT)
.legalForCartesianProduct(allIntScalars, allPtrs)
@@ -396,6 +408,15 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
return SrcTy.isPointerVector() && DstTy.isVector() &&
!DstTy.isPointer() &&
DstTy.getNumElements() == SrcTy.getNumElements();
+ })
+ .legalIf(sameSize(0, 1))
+ .widenScalarIf(smallerThan(0, 1),
+ [](const LegalityQuery &Query) {
+ return std::pair(
+ 0, LLT::scalar(Query.Types[1].getSizeInBits()));
+ })
+ .narrowScalarIf(largerThan(0, 1), [](const LegalityQuery &Query) {
+ return std::pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
});
getActionDefinitionsBuilder(G_PTR_ADD)
.legalForCartesianProduct(allPtrs, allIntScalars)
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.h b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.h
index d7bb31206c2e2..5410105c0fdf9 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.h
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.h
@@ -20,6 +20,7 @@ namespace llvm {
class LLVMContext;
class SPIRVSubtarget;
+class SPIRVTargetMachine;
// This class provides the information for legalizing SPIR-V instructions.
class SPIRVLegalizerInfo : public LegalizerInfo {
@@ -32,7 +33,7 @@ class SPIRVLegalizerInfo : public LegalizerInfo {
bool legalizeIntrinsic(LegalizerHelper &Helper,
MachineInstr &MI) const override;
- SPIRVLegalizerInfo(const SPIRVSubtarget &ST);
+ SPIRVLegalizerInfo(const SPIRVSubtarget &ST, const TargetMachine &TM);
private:
bool legalizeIsFPClass(LegalizerHelper &Helper, MachineInstr &MI,
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 75cc7133f1766..9a33a5f8a2446 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -366,10 +366,11 @@ static SPIRVTypeInst propagateSPIRVType(MachineInstr *MI,
if (SpvType) {
// check if the address space needs correction
LLT RegType = MRI.getType(Reg);
+ const Triple &TT = MI->getMF()->getTarget().getTargetTriple();
if (SpvType->getOpcode() == SPIRV::OpTypePointer &&
RegType.isPointer() &&
- storageClassToAddressSpace(GR->getPointerStorageClass(SpvType)) !=
- RegType.getAddressSpace()) {
+ storageClassToAddressSpace(GR->getPointerStorageClass(SpvType),
+ TT) != RegType.getAddressSpace()) {
const SPIRVSubtarget &ST =
MI->getParent()->getParent()->getSubtarget<SPIRVSubtarget>();
auto TSC = addressSpaceToStorageClass(RegType.getAddressSpace(), ST);
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 916353ffdf381..64503d1d74bdb 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -38,6 +38,7 @@
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
#include <regex>
+#include <utility>
using namespace llvm;
@@ -115,7 +116,8 @@ static bool lowerIntrinsicToFunction(IntrinsicInst *Intrinsic,
// cases we wrap the intrinsic in @spirv.llvm_memset_* function and expand the
// intrinsic to a loop via expandMemSetAsLoop().
if (auto *MSI = dyn_cast<MemSetInst>(Intrinsic))
- if (isa<Constant>(MSI->getValue()) && isa<ConstantInt>(MSI->getLength()))
+ if (!MSI->isForceInlined() && isa<Constant>(MSI->getValue()) &&
+ isa<ConstantInt>(MSI->getLength()))
return false; // It is handled later using OpCopyMemorySized.
Module *M = Intrinsic->getModule();
@@ -450,10 +452,14 @@ bool SPIRVPrepareFunctionsImpl::substituteIntrinsicCalls(Function *F) {
if (!CF || !CF->isIntrinsic())
continue;
auto *II = cast<IntrinsicInst>(Call);
- if (Intrinsic::isTargetIntrinsic(II->getIntrinsicID()) &&
- II->getCalledOperand()->getName().starts_with("llvm.spv"))
+ unsigned IID = II->getIntrinsicID();
+ if (Intrinsic::isTargetIntrinsic(IID) &&
+ II->getCalledOperand()->getName().starts_with("llvm.spv")) {
+ if (IID == Intrinsic::spv_opaque_ptr_cast)
+ Changed |= lowerIntrinsicToFunction(II, TTI);
continue;
- switch (II->getIntrinsicID()) {
+ }
+ switch (IID) {
case Intrinsic::memset:
case Intrinsic::bswap:
Changed |= lowerIntrinsicToFunction(II, TTI);
@@ -805,6 +811,64 @@ bool SPIRVPrepareFunctionsImpl::removeAggregateTypesFromCalls(Function *F) {
return true;
}
+static inline bool isTargetSpecificASCast(unsigned SrcAS, unsigned DstAS,
+ const Triple &TT) {
+ static const unsigned GenericAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::Generic, TT);
+ static const unsigned BufferFatPointerAS = AMDGPUAS::BUFFER_FAT_POINTER;
+ static const unsigned BufferResourceAS = AMDGPUAS::BUFFER_RESOURCE;
+ static const unsigned UniformConstAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::UniformConstant, TT);
+
+ static const std::pair<unsigned, unsigned> Casts[] = {
+ {GenericAS, BufferFatPointerAS}, {GenericAS, BufferResourceAS},
+ {BufferFatPointerAS, GenericAS}, {BufferFatPointerAS, BufferResourceAS},
+ {BufferResourceAS, GenericAS}, {BufferResourceAS, BufferFatPointerAS},
+ {GenericAS, UniformConstAS}, {UniformConstAS, GenericAS}
+ };
+
+ return find(Casts, std::make_pair(SrcAS, DstAS));
+}
+
+static bool substituteInvalidAddrSpaceCasts(Function *F) {
+ // AMDGPU supports a superset of the AS casts allowed by SPIR-V, specifically
+ // - casts to/from UniformConstant from/to Generic
+ // - casts to/from Input from/to Output (these are actually buffer specific
+ // ASes that we preserve)
+ // - casts to/from Input or Output from/to Generic (see previous)
+ // We handle these by replacing the cast instruction with a stub function,
+ // which is subsequently handled during run time finalisation.
+ SmallVector<std::reference_wrapper<Instruction>> ToSubstitute;
+ copy_if(instructions(F), std::back_inserter(ToSubstitute), [F](auto &&I) {
+ if (!isa<AddrSpaceCastInst>(I))
+ return false;
+
+ auto &ASC = cast<AddrSpaceCastInst>(I);
+
+ return isTargetSpecificASCast(ASC.getSrcAddressSpace(),
+ ASC.getDestAddressSpace(),
+ F->getParent()->getTargetTriple());
+ });
+
+ if (ToSubstitute.empty())
+ return false;
+
+ IRBuilder<> B(F->getContext());
+ for_each(ToSubstitute, [&B](auto &&I) {
+ auto &ASC = I.get();
+
+ B.SetInsertPoint(&ASC);
+ CallInst *TASC = B.CreateIntrinsic(ASC.getType(),
+ Intrinsic::spv_opaque_ptr_cast,
+ {ASC.getOperand(0)});
+
+ ASC.replaceAllUsesWith(TASC);
+ ASC.eraseFromParent();
+ });
+
+ return true;
+}
+
bool SPIRVPrepareFunctionsImpl::runOnModule(Module &M) {
// Resolve the SPIR-V environment from module content before any
// function-level processing. This must happen before legalization so that
@@ -814,7 +878,8 @@ bool SPIRVPrepareFunctionsImpl::runOnModule(Module &M) {
->resolveEnvFromModule(M);
bool Changed = false;
- if (M.functions().empty()) {
+ if (M.functions().empty() ||
+ all_of(M, [](auto &&F) { return F.isDeclaration(); })) {
// If there are no functions, insert a service
// function so that the global/constant tracking intrinsics
// will be created. Without these intrinsics the generated SPIR-V
@@ -830,6 +895,8 @@ bool SPIRVPrepareFunctionsImpl::runOnModule(Module &M) {
Changed |= terminateBlocksAfterTrap(M, Intrinsic::ubsantrap);
for (Function &F : M) {
+ if (M.getTargetTriple().getVendor() == llvm::Triple::AMD)
+ Changed |= substituteInvalidAddrSpaceCasts(&F);
Changed |= substituteAbortKHRCalls(&F);
Changed |= substituteIntrinsicCalls(&F);
Changed |= sortBlocks(F);
diff --git a/llvm/lib/Target/SPIRV/SPIRVPushConstantAccess.cpp b/llvm/lib/Target/SPIRV/SPIRVPushConstantAccess.cpp
index 31e69a49a1ed1..7a5309f31f6b8 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPushConstantAccess.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPushConstantAccess.cpp
@@ -33,7 +33,8 @@ static bool replacePushConstantAccesses(Module &M, SPIRVGlobalRegistry *GR) {
bool Changed = false;
for (GlobalVariable &GV : make_early_inc_range(M.globals())) {
if (GV.getAddressSpace() !=
- storageClassToAddressSpace(SPIRV::StorageClass::PushConstant))
+ storageClassToAddressSpace(SPIRV::StorageClass::PushConstant,
+ M.getTargetTriple()))
continue;
convertUsersOfConstantsToInstructions(
diff --git a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
index 5619748965112..f80d02c0df293 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
@@ -54,7 +54,9 @@ SPIRVSubtarget::SPIRVSubtarget(const Triple &TT, const std::string &CPU,
const std::string &FS,
const SPIRVTargetMachine &TM)
: SPIRVGenSubtargetInfo(TT, CPU, /*TuneCPU=*/CPU, FS),
- PointerSize(TM.getPointerSizeInBits(/* AS= */ 0)),
+ PointerSize(TM.getPointerSizeInBits(
+ /* AS= */ storageClassToAddressSpace(SPIRV::StorageClass::Generic,
+ TM.getTargetTriple()))),
InstrInfo(initSubtargetDependencies(CPU, FS)), FrameLowering(*this),
TLInfo(TM, *this), TargetTriple(TT) {
switch (TT.getSubArch()) {
@@ -112,7 +114,7 @@ SPIRVSubtarget::SPIRVSubtarget(const Triple &TT, const std::string &CPU,
GR = std::make_unique<SPIRVGlobalRegistry>(TM.createDataLayout());
CallLoweringInfo = std::make_unique<SPIRVCallLowering>(TLInfo, GR.get());
InlineAsmInfo = std::make_unique<SPIRVInlineAsmLowering>(TLInfo);
- Legalizer = std::make_unique<SPIRVLegalizerInfo>(*this);
+ Legalizer = std::make_unique<SPIRVLegalizerInfo>(*this, TM);
RegBankInfo = std::make_unique<SPIRVRegisterBankInfo>();
InstSelector.reset(createSPIRVInstructionSelector(TM, *this, *RegBankInfo));
}
@@ -189,7 +191,8 @@ void SPIRVSubtarget::setEnv(SPIRVEnvType E) {
// Reinitialize Env-dependent state aka ExtInstSet and legalizer info.
initAvailableExtInstSets();
- Legalizer = std::make_unique<SPIRVLegalizerInfo>(*this);
+ Legalizer = std::make_unique<SPIRVLegalizerInfo>(*this,
+ TLInfo.getTargetMachine());
}
void SPIRVSubtarget::resolveEnvFromModule(const Module &M) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 0ef31f4182b4e..96262db38c46e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -389,9 +389,13 @@ addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI) {
? SPIRV::StorageClass::HostOnlyINTEL
: SPIRV::StorageClass::CrossWorkgroup;
case 7:
- return SPIRV::StorageClass::Input;
+ return STI.getTargetTriple().getVendor() == Triple::VendorType::AMD
+ ? SPIRV::StorageClass::DeviceOnlyINTEL
+ : SPIRV::StorageClass::Input;
case 8:
- return SPIRV::StorageClass::Output;
+ return STI.getTargetTriple().getVendor() == Triple::VendorType::AMD
+ ? SPIRV::StorageClass::HostOnlyINTEL
+ : SPIRV::StorageClass::Output;
case 9:
return SPIRV::StorageClass::CodeSectionINTEL;
case 10:
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index 27b196cb8dad3..c5c3be94c9f4e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -247,7 +247,8 @@ constexpr bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) {
// TODO: maybe the following two functions should be handled in the subtarget
// to allow for different OpenCL vs Vulkan handling.
constexpr unsigned
-storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC) {
+storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC,
+ const Triple& TT) {
switch (SC) {
case SPIRV::StorageClass::Function:
return 0;
@@ -260,8 +261,12 @@ storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC) {
case SPIRV::StorageClass::Generic:
return 4;
case SPIRV::StorageClass::DeviceOnlyINTEL:
+ if (TT.getVendor() == Triple::VendorType::AMD)
+ return AMDGPUAS::BUFFER_FAT_POINTER;
return 5;
case SPIRV::StorageClass::HostOnlyINTEL:
+ if (TT.getVendor() == Triple::VendorType::AMD)
+ return AMDGPUAS::BUFFER_RESOURCE;
return 6;
case SPIRV::StorageClass::Input:
return 7;
diff --git a/llvm/lib/TargetParser/TargetDataLayout.cpp b/llvm/lib/TargetParser/TargetDataLayout.cpp
index a2125eeb82932..22fec2f2ef572 100644
--- a/llvm/lib/TargetParser/TargetDataLayout.cpp
+++ b/llvm/lib/TargetParser/TargetDataLayout.cpp
@@ -487,9 +487,14 @@ static std::string computeSPIRVDataLayout(const Triple &TT) {
if (Arch == Triple::spirv)
return "e-ve-i64:64-n8:16:32:64-G10";
if (TT.getVendor() == Triple::VendorType::AMD &&
- TT.getOS() == Triple::OSType::AMDHSA)
- return "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-"
- "v512:512-v1024:1024-n32:64-S32-G1-P4-A0";
+ TT.getOS() == Triple::OSType::AMDHSA) {
+ auto DL = computeAMDDataLayout(TT);
+ DL.replace(DL.find("p:64:64"), 7, "p:32:32"); // AS0 is Function
+ DL.replace(DL.find("p2:32:32"), 8, "p2:64:64"); // AS2 is UniformConstant
+ DL.replace(DL.find("A5"), 2, "A0"); // AllocaAS is Function
+ DL.insert(DL.find_last_of('-') + 1, "P4-"); // ProgramAS is Generic
+ return DL;
+ }
if (TT.getVendor() == Triple::VendorType::Intel)
return "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-"
"v512:512-v1024:1024-n8:16:32:64-G1-P9-A0";
diff --git a/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-calls.ll b/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-calls.ll
new file mode 100644
index 0000000000000..2396ae86ce0ef
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-calls.ll
@@ -0,0 +1,81 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+target triple = "spirv64-amd-amdhsa"
+
+; CHECK: OpName %[[#RECUR_INNER_1:]] "recur.inner.1"
+; CHECK: OpName %[[#RECUR_INNER_2:]] "recur.inner.2"
+; CHECK: OpName %[[#RECUR_OUTER:]] "recur.outer"
+; CHECK: OpName %[[#CALLER:]] "caller"
+; CHECK: OpName %[[#FOO:]] "foo"
+; CHECK: %[[#INT32_TY:]] = OpTypeInt 32
+; CHECK: %[[#I32PTR_ADDRSPACE_7:]] = OpTypePointer DeviceOnlyINTEL %[[#INT32_TY]]
+; CHECK: %[[#INT8_TY:]] = OpTypeInt 8
+; CHECK: %[[#I8PTR_ADDRSPACE_7:]] = OpTypePointer DeviceOnlyINTEL %[[#INT8_TY]]
+
+; CHECK: %[[#RECUR_INNER_1]] = OpFunction %[[#I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#X:]] = OpFunctionParameter %[[#I32PTR_ADDRSPACE_7]]
+; CHECK: %[[#RET0:]] = OpBitcast %[[#I8PTR_ADDRSPACE_7]] %[[#X]]
+; CHECK: %[[#RET1:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#RECUR_INNER_2]] %[[#]] %[[#X]]
+; CHECK: %[[#RET:]] = OpPhi %[[#I8PTR_ADDRSPACE_7]] %[[#RET1]] %[[#]] %[[#RET0]] %[[#]]
+; CHECK: OpReturnValue %[[#RET]]
+define spir_func ptr addrspace(7) @recur.inner.1(ptr addrspace(7) %x, i32 %v) {
+bb:
+ %isBase = icmp sgt i32 %v, 0
+ br i1 %isBase, label %recur, label %else
+recur:
+ %dec = sub i32 %v, 1
+ %inc = call ptr addrspace(7) @recur.inner.2(i32 %dec, ptr addrspace(7) %x)
+ br label %end
+else:
+ br label %end
+end:
+ %ret = phi ptr addrspace(7) [%inc, %recur], [%x, %else]
+ ret ptr addrspace(7) %ret
+}
+
+; CHECK: %[[#RECUR_INNER_2]] = OpFunction %[[#I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#X1:]] = OpFunctionParameter %[[#I32PTR_ADDRSPACE_7]]
+; CHECK: %[[#GEP:]] = OpPtrAccessChain %[[#I32PTR_ADDRSPACE_7]] %[[#X1]]
+; CHECK: %[[#RET2:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#RECUR_INNER_1]] %[[#GEP]]
+; CHECK: OpReturnValue %[[#RET2]]
+define spir_func ptr addrspace(7) @recur.inner.2(i32 %v, ptr addrspace(7) %x) {
+ %inc = getelementptr i32, ptr addrspace(7) %x, i32 1
+ %ret = call ptr addrspace(7) @recur.inner.1(ptr addrspace(7) %inc, i32 %v)
+ ret ptr addrspace(7) %ret
+}
+
+; CHECK: %[[#RECUR_OUTER]] = OpFunction
+; CHECK: %[[#X2:]] = OpFunctionParameter %[[#I32PTR_ADDRSPACE_7]]
+; CHECK: %[[#ARG:]] = OpFunctionParameter %[[#I32PTR_GENERIC:]]
+; CHECK: %[[#LD:]] = OpLoad %[[#INT32_TY]] %[[#ARG]]
+; CHECK: %[[#STV:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#RECUR_INNER_1]] %[[#X2]] %[[#LD]]
+; CHECK: %[[#ARG_AS_I8PTR_ADDRSPACE_7PTR_GENERIC:]] = OpBitcast %[[#I8PTR_ADDRSPACE_7PTR_GENERIC:]] %[[#ARG]]
+; CHECK: OpStore %[[#ARG_AS_I8PTR_ADDRSPACE_7PTR_GENERIC]] %[[#STV]]
+define spir_func void @recur.outer(ptr addrspace(7) %x, ptr addrspace(4) %arg) {
+ %bound = load i32, ptr addrspace(4) %arg
+ %ret = call ptr addrspace(7) @recur.inner.1(ptr addrspace(7) %x, i32 %bound)
+ store ptr addrspace(7) %ret, ptr addrspace(4) %arg
+ ret void
+}
+
+; CHECK: %[[#CALLER]] = OpFunction
+; CHECK: %[[#ARG1:]] = OpFunctionParameter %[[#I8PTR_ADDRSPACE_7PTR_ADDRSPACE_7:]]
+; CHECK: %[[#ARG1_AS_I8PTR_ADDRSPACE_7:]] = OpBitcast %[[#I8PTR_ADDRSPACE_7]] %[[#ARG1]]
+; CHECK: %[[#STV:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#EXTERN:]] %[[#ARG1_AS_I8PTR_ADDRSPACE_7]]
+; CHECK: OpStore %[[#ARG1]] %[[#STV]]
+declare spir_func ptr addrspace(7) @extern(ptr addrspace(7) %arg)
+define spir_func void @caller(ptr addrspace(7) noundef nonnull %arg) {
+ %v = call ptr addrspace(7) @extern(ptr addrspace(7) %arg)
+ store ptr addrspace(7) %v, ptr addrspace(7) %arg
+ ret void
+}
+
+; CHECK: %[[#FOO]] = OpFunction %[[#I32PTR_ADDRSPACE_7]]
+; CHECK: %[[#ARG2:]] = OpFunctionParameter %[[#I32PTR_ADDRSPACE_7]]
+; CHECK: %[[#RET3:]] = OpInBoundsPtrAccessChain %[[#I32PTR_ADDRSPACE_7]] %[[#ARG2]]
+; CHECK: OpReturnValue %[[#RET3]]
+define spir_func ptr addrspace(7) @foo(ptr addrspace(7) noalias noundef nonnull %arg) {
+ %ret = getelementptr inbounds i32, ptr addrspace(7) %arg, i32 1
+ ret ptr addrspace(7) %ret
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-constants.ll b/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-constants.ll
new file mode 100644
index 0000000000000..41f1904e2157e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-constants.ll
@@ -0,0 +1,305 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+target triple = "spirv64-amd-amdhsa"
+
+ at buf = external addrspace(8) global i8
+ at flat = external addrspace(1) global i8
+
+; CHECK: OpName %[[#NULL:]] "null"
+; CHECK: OpName %[[#NULL_VECTOR:]] "null_vector"
+; CHECK: OpName %[[#POISON:]] "poison"
+; CHECK: OpName %[[#POISON_VEC:]] "poison_vec"
+; CHECK: OpName %[[#CAST_GLOBAL:]] "cast_global"
+; CHECK: OpName %[[#BUF:]] "buf"
+; CHECK: OpName %[[#OPAQUE_PTR_CAST_P8_P7:]] "spirv.llvm_spv_opaque_ptr_cast_p7_p8"
+; CHECK: OpName %[[#CAST_NULL:]] "cast_null"
+; CHECK: OpName %[[#CAST_VEC:]] "cast_vec"
+; CHECK: OpName %[[#GEP:]] "gep"
+; CHECK: OpName %[[#GEP_VECTOR:]] "gep_vector"
+; CHECK: OpName %[[#GEP_OF_P7:]] "gep_of_p7"
+; CHECK: OpName %[[#FLAT:]] "flat"
+; CHECK: OpName %[[#GEP_OF_P7_VECTOR:]] "gep_of_p7_vector"
+; CHECK: OpName %[[#GEP_OF_P7_STRUCT:]] "gep_of_p7_struct"
+; CHECK: OpName %[[#GEP_P7_FROM_P7:]] "gep_p7_from_p7"
+; CHECK: OpName %[[#PTRTOINT:]] "ptrtoint"
+; CHECK: OpName %[[#PTRTOINT_LONG:]] "ptrtoint_long"
+; CHECK: OpName %[[#PTRTOINT_SHORT:]] "ptrtoint_short"
+; CHECK: OpName %[[#PTRTOINT_VERY_SHORT:]] "ptrtoint_very_short"
+; CHECK: OpName %[[#PTRTOINT_VEC:]] "ptrtoint_vec"
+; CHECK: OpName %[[#INTTOPTR:]] "inttoptr"
+; CHECK: OpName %[[#INTTOPTR_VEC:]] "inttoptr_vec"
+; CHECK: OpName %[[#FANCY_ZERO:]] "fancy_zero"
+; CHECK: OpName %[[#LOAD_NULL:]] "load_null"
+; CHECK: OpName %[[#STORE_NULL:]] "store_null"
+; CHECK: OpName %[[#LOAD_POISON:]] "load_poison"
+; CHECK: OpName %[[#STORE_POISON:]] "store_poison"
+
+; CHECK: %[[#INT8_TY:]] = OpTypeInt 8
+; CHECK: %[[#I8PTR_ADDRSPACE_7:]] = OpTypePointer DeviceOnlyINTEL %[[#INT8_TY]]
+; CHECK: %[[#INT32_TY:]] = OpTypeInt 32
+; CHECK: %[[#I32_PTR_GLOBAL:]] = OpTypePointer CrossWorkgroup %[[#INT32_TY]]
+; CHECK: %[[#NULLPTR_I8PTR_ADDRSPACE_7:]] = OpConstantNull %[[#I8PTR_ADDRSPACE_7]]
+ ; %8 = OpConstantNull %5
+ ; %9 = OpVariable %6 CrossWorkgroup %8
+; CHECK: %[[#VEC2_I8PTR_ADDRSPACE_7:]] = OpTypeVector %[[#I8PTR_ADDRSPACE_7]] 2
+; CHECK: %[[#INT160_TY:]] = OpTypeInt 160
+; CHECK: %[[#ZERO_INT160:]] = OpConstantNull %[[#INT160_TY]]
+; CHECK: %[[#NULLVEC2_I8PTR_ADDRSPACE_7:]] = OpConstantComposite %[[#VEC2_I8PTR_ADDRSPACE_7]] %[[#ZERO_INT160]] %[[#ZERO_INT160]]
+; CHECK: %[[#UNDEF_I8PTR_ADDRSPACE_7:]] = OpUndef %[[#I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#UNDEF_VEC2_I8PTR_ADDRSPACE_7:]] = OpUndef %[[#VEC2_I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#I8PTR_ADDRSPACE_8:]] = OpTypePointer HostOnlyINTEL %[[#INT8_TY]]
+ ; %18 = OpTypeFunction %3 %17
+; CHECK: %[[#BUF:]] = OpVariable %[[#I8PTR_ADDRSPACE_8]] HostOnlyINTEL
+; CHECK: %[[#NULLPTR_I8PTR_ADDRSPACE_8:]] = OpConstantNull %[[#I8PTR_ADDRSPACE_8]]
+; CHECK: %[[#I32PTR_ADDRSPACE_7:]] = OpTypePointer DeviceOnlyINTEL %[[#INT32_TY]]
+ ; %24 = OpTypeFunction %23
+ ; %25 = OpConstant %5 4
+; CHECK: %[[#ARR_TY:]] = OpTypeArray %[[#INT32_TY]]
+; CHECK: %[[#ARRPTR_ADDRSPACE_7:]] = OpTypePointer DeviceOnlyINTEL %[[#ARR_TY]]
+; CHECK: %[[#INT64_TY:]] = OpTypeInt 64
+ ; %29 = OpConstant %5 1
+ ; %30 = OpConstant %28 2
+; CHECK: %[[#VEC2_I32:]] = OpTypeVector %[[#INT32_TY]] 2
+ ; %32 = OpConstant %5 3
+; CHECK: %[[#CONSTEXPR_CAST_PTR_AS8_TO_INT:]] = OpSpecConstantOp %[[#INT64_TY]] ConvertPtrToU %[[#BUF]]
+; CHECK: %[[#CONSTEXPR_CAST_INT_AS8_PTR_TO_PTR_AS7:]] = OpSpecConstantOp %[[#I8PTR_ADDRSPACE_7]] ConvertUToPtr %[[#CONSTEXPR_CAST_PTR_AS8_TO_INT]]
+ ; %35 = OpConstantComposite %31 %32 %29
+; CHECK: %[[#TEMP_VEC_WITH_CONSTEXPR_AS_CAST:]] = OpSpecConstantComposite %[[#VEC2_I32]] %[[#CONSTEXPR_CAST_INT_AS8_PTR_TO_PTR_AS7]]
+; CHECK: %[[#I8PTR_ADDRSPACE_7PTR_GLOBAL:]] = OpTypePointer CrossWorkgroup %[[#I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#I8PTR_GLOBAL:]] = OpTypePointer CrossWorkgroup %[[#INT8_TY]]
+; CHECK: %[[#FLAT:]] = OpVariable %[[#I8PTR_GLOBAL]] CrossWorkgroup
+; CHECK: %[[#VEC2_I8PTR_ADDRSPACE_7PTR_GLOBAL:]] = OpTypePointer CrossWorkgroup %[[#VEC2_I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#STRUCT_TY:]] = OpTypeStruct %[[#I8PTR_ADDRSPACE_7]] %[[#INT32_TY]]
+; CHECK: %[[#STRUCTPTR_GLOBAL:]] = OpTypePointer CrossWorkgroup %[[#STRUCT_TY]]
+; CHECK: %[[#PTRI8PTR_ADDRSPACE_7_ADDRSPACE_7:]] = OpTypePointer DeviceOnlyINTEL %[[#I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#INT256_TY:]] = OpTypeInt 256
+; CHECK: %[[#VEC2_I160_TY:]] = OpTypeVector %[[#INT160_TY]] 2
+; CHECK: %[[#NULL_VEC2_I160:]] = OpConstantNull %[[#VEC2_I160_TY]]
+; CHECK: %[[#CONSTEXPR_CAST_INT_1_TO_PTR_AS7:]] = OpSpecConstantOp %3 ConvertUToPtr
+; CHECK: %[[#CONSTEXPR_CAST_INT_2_TO_PTR_AS7:]] = OpSpecConstantOp %3 ConvertUToPtr
+
+; CHECK: %[[#NULL]] = OpFunction %[[#I8PTR_ADDRSPACE_7]]
+; CHECK: OpReturnValue %[[#NULLPTR_I8PTR_ADDRSPACE_7]]
+define spir_func ptr addrspace(7) @null() {
+ ret ptr addrspace(7) null
+}
+
+; CHECK: %[[#NULL_VECTOR]] = OpFunction %[[#VEC2_I8PTR_ADDRSPACE_7]]
+; CHECK: OpReturnValue %[[#NULLVEC2_I8PTR_ADDRSPACE_7]]
+define spir_func <2 x ptr addrspace(7)> @null_vector() {
+ ret <2 x ptr addrspace(7)> zeroinitializer
+}
+
+; CHECK: %[[#POISON]] = OpFunction %[[#I8PTR_ADDRSPACE_7]]
+; CHECK: OpReturnValue %[[#UNDEF_I8PTR_ADDRSPACE_7]]
+define spir_func ptr addrspace(7) @poison() {
+ ret ptr addrspace(7) poison
+}
+
+; CHECK: %[[#POISON_VEC]] = OpFunction %[[#VEC2_I8PTR_ADDRSPACE_7]]
+; CHECK: OpReturnValue %[[#UNDEF_VEC2_I8PTR_ADDRSPACE_7]]
+define spir_func <2 x ptr addrspace(7)> @poison_vec() {
+ ret <2 x ptr addrspace(7)> poison
+}
+
+; CHECK: %[[#CAST_GLOBAL]] = OpFunction %[[#I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#CAST_FROM_AS8_TO_AS7:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST_P8_P7]] %[[#BUF]]
+; CHECK: OpReturnValue %[[#CAST_FROM_AS8_TO_AS7]]
+define spir_func ptr addrspace(7) @cast_global() {
+ ret ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7))
+}
+
+; CHECK: %[[#CAST_NULL]] = OpFunction %[[#I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#CAST_NULL_FROM_AS8_TO_AS7:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST_P8_P7]] %[[#NULLPTR_I8PTR_ADDRSPACE_8]]
+; CHECK: OpReturnValue %[[#CAST_NULL_FROM_AS8_TO_AS7]]
+define spir_func ptr addrspace(7) @cast_null() {
+ ret ptr addrspace(7) addrspacecast (ptr addrspace(8) null to ptr addrspace(7))
+}
+
+; CHECK: %[[#CAST_VEC]] = OpFunction %[[#VEC2_I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#AS8_TO_AS7:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST_P8_P7]] %[[#BUF]]
+; CHECK: %[[#NULL_AS8_TO_AS7:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST_P8_P7]] %[[#NULLPTR_I8PTR_ADDRSPACE_8]]
+; CHECK: %[[#V0:]] = OpCompositeInsert %[[#VEC2_I8PTR_ADDRSPACE_7]] %[[#AS8_TO_AS7]] %[[#UNDEF_VEC2_I8PTR_ADDRSPACE_7]] 0
+; CHECK: %[[#V:]] = OpCompositeInsert %[[#VEC2_I8PTR_ADDRSPACE_7]] %[[#NULL_AS8_TO_AS7]] %[[#V0]] 1
+; CHECK: OpReturnValue %[[#V]]
+define spir_func <2 x ptr addrspace(7)> @cast_vec() {
+ ret <2 x ptr addrspace(7)> addrspacecast (
+ <2 x ptr addrspace(8)> <ptr addrspace(8) @buf, ptr addrspace(8) null>
+ to <2 x ptr addrspace(7)>)
+}
+
+; CHECK: %[[#GEP]] = OpFunction %[[#I32PTR_ADDRSPACE_7]]
+; CHECK: %[[#BUF_FROM_AS8_TO_AS7:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST_P8_P7]] %[[#BUF]]
+; CHECK: %[[#BC_TO_ARR:]] = OpBitcast %[[#ARRPTR_ADDRSPACE_7]] %[[#BUF_FROM_AS8_TO_AS7]]
+; CHECK: %[[#GEP_RESULT:]] = OpInBoundsPtrAccessChain %[[#I32PTR_ADDRSPACE_7]] %[[#BC_TO_ARR]]
+; CHECK: OpReturnValue %[[#GEP_RESULT]]
+define spir_func ptr addrspace(7) @gep() {
+ ret ptr addrspace(7) getelementptr inbounds (
+ [4 x i32],
+ ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)),
+ i64 2, i32 1)
+}
+
+; CHECK: %[[#GEP_VECTOR]] = OpFunction %[[#VEC2_I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#GEP_VEC_RESULT:]] = OpPtrAccessChain %[[#VEC2_I8PTR_ADDRSPACE_7]] %[[#TEMP_VEC_WITH_CONSTEXPR_AS_CAST]]
+; CHECK: OpReturnValue %[[#GEP_VEC_RESULT]]
+define spir_func <2 x ptr addrspace(7)> @gep_vector() {
+ ret <2 x ptr addrspace(7)> getelementptr (
+ <2 x i32>,
+ <2 x ptr addrspace(7)>
+ <ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)),
+ ptr addrspace(7) null>,
+ <2 x i32> <i32 3, i32 1>)
+}
+
+; CHECK: %[[#GEP_OF_P7]] = OpFunction %[[#I8PTR_ADDRSPACE_7PTR_GLOBAL]]
+; CHECK: %[[#BC0:]] = OpBitcast %[[#I8PTR_ADDRSPACE_7PTR_GLOBAL]] %[[#FLAT]]
+; CHECK: %[[#GEP_P7:]] = OpInBoundsPtrAccessChain %[[#I8PTR_ADDRSPACE_7PTR_GLOBAL]] %[[#BC0]]
+; CHECK: OpReturnValue %[[#GEP_P7]]
+define spir_func ptr addrspace(1) @gep_of_p7() {
+ ret ptr addrspace(1) getelementptr inbounds (ptr addrspace(7), ptr addrspace(1) @flat, i64 2)
+}
+
+; CHECK: %[[#GEP_OF_P7_VECTOR]] = OpFunction %[[#VEC2_I8PTR_ADDRSPACE_7PTR_GLOBAL]]
+; CHECK: %[[#BC1:]] = OpBitcast %[[#VEC2_I8PTR_ADDRSPACE_7PTR_GLOBAL]] %[[#FLAT]]
+; CHECK: %[[#GEP_P7_VEC:]] = OpPtrAccessChain %[[#VEC2_I8PTR_ADDRSPACE_7PTR_GLOBAL]] %[[#BC1]]
+; CHECK: OpReturnValue %[[#GEP_P7_VEC]]
+define spir_func ptr addrspace(1) @gep_of_p7_vector() {
+ ret ptr addrspace(1) getelementptr (<2 x ptr addrspace(7)>, ptr addrspace(1) @flat, i64 2)
+}
+
+; CHECK: %[[#GEP_OF_P7_STRUCT]] = OpFunction %[[#STRUCTPTR_GLOBAL]]
+; CHECK: %[[#BC2:]] = OpBitcast %[[#STRUCTPTR_GLOBAL]] %[[#FLAT]]
+; CHECK: %[[#GEP_P7_STRUCT:]] = OpPtrAccessChain %[[#STRUCTPTR_GLOBAL]] %[[#BC2]]
+; CHECK: OpReturnValue %[[#GEP_P7_STRUCT]]
+define spir_func ptr addrspace(1) @gep_of_p7_struct() {
+ ret ptr addrspace(1) getelementptr ({ptr addrspace(7), i32}, ptr addrspace(1) @flat, i64 2)
+}
+
+; CHECK: %[[#GEP_P7_FROM_P7]] = OpFunction %[[#PTRI8PTR_ADDRSPACE_7_ADDRSPACE_7]]
+; CHECK: %[[#BUF_AS8_TO_AS7:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST_P8_P7]] %[[#BUF]]
+; CHECK: %[[#BC3:]] = OpBitcast %[[#PTRI8PTR_ADDRSPACE_7_ADDRSPACE_7]] %[[#BUF_AS8_TO_AS7]]
+; CHECK: %[[#GEP_P7_P7:]] = OpPtrAccessChain %[[#PTRI8PTR_ADDRSPACE_7_ADDRSPACE_7]] %[[#BC3]]
+; CHECK: OpReturnValue %[[#GEP_P7_P7]]
+define spir_func ptr addrspace(7) @gep_p7_from_p7() {
+ ret ptr addrspace(7) getelementptr (ptr addrspace(7),
+ ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)),
+ i64 2)
+}
+
+; CHECK: %[[#PTRTOINT]] = OpFunction %[[#INT160_TY]]
+; CHECK: %[[#BUF_AS8_TO_AS7_1:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST_P8_P7]] %[[#BUF]]
+; CHECK: %[[#BC4:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#BUF_AS8_TO_AS7_1]]
+; CHECK: %[[#PTR:]] = OpPtrAccessChain %[[#I32PTR_ADDRSPACE_7]] %[[#BC4]]
+; CHECK: %[[#PTR_TO_U160:]] = OpConvertPtrToU %[[#INT160_TY]] %[[#PTR]]
+; CHECK: OpReturnValue %[[#PTR_TO_U160]]
+define spir_func i160 @ptrtoint() {
+ ret i160 ptrtoint(
+ ptr addrspace(7) getelementptr(
+ i32, ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)),
+ i32 3) to i160)
+}
+
+; CHECK: %[[#PTRTOINT_LONG]] = OpFunction %[[#INT256_TY]]
+; CHECK: %[[#BUF_AS8_TO_AS7_2:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST_P8_P7]] %[[#BUF]]
+; CHECK: %[[#BC5:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#BUF_AS8_TO_AS7_2]]
+; CHECK: %[[#PTR1:]] = OpPtrAccessChain %[[#I32PTR_ADDRSPACE_7]] %[[#BC5]]
+; CHECK: %[[#PTR_TO_U256:]] = OpConvertPtrToU %[[#INT256_TY]] %[[#PTR1]]
+; CHECK: OpReturnValue %[[#PTR_TO_U256]]
+define spir_func i256 @ptrtoint_long() {
+ ret i256 ptrtoint(
+ ptr addrspace(7) getelementptr(
+ i32, ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)),
+ i32 3) to i256)
+}
+
+; CHECK: %[[#PTRTOINT_SHORT]] = OpFunction %[[#INT64_TY]]
+; CHECK: %[[#BUF_AS8_TO_AS7_3:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST_P8_P7]] %[[#BUF]]
+; CHECK: %[[#BC6:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#BUF_AS8_TO_AS7_3]]
+; CHECK: %[[#PTR2:]] = OpPtrAccessChain %[[#I32PTR_ADDRSPACE_7]] %[[#BC6]]
+; CHECK: %[[#PTR_TO_U64:]] = OpConvertPtrToU %[[#INT64_TY]] %[[#PTR2]]
+; CHECK: OpReturnValue %[[#PTR_TO_U64]]
+define spir_func i64 @ptrtoint_short() {
+ ret i64 ptrtoint(
+ ptr addrspace(7) getelementptr(
+ i32, ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)),
+ i32 3) to i64)
+}
+
+; CHECK: %[[#PTRTOINT_VERY_SHORT]] = OpFunction %[[#INT32_TY]]
+; CHECK: %[[#BUF_AS8_TO_AS7_4:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST_P8_P7]] %[[#BUF]]
+; CHECK: %[[#BC7:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#BUF_AS8_TO_AS7_4]]
+; CHECK: %[[#PTR3:]] = OpPtrAccessChain %[[#I32PTR_ADDRSPACE_7]] %[[#BC7]]
+; CHECK: %[[#PTR_TO_U32:]] = OpConvertPtrToU %[[#INT32_TY]] %[[#PTR3]]
+; CHECK: OpReturnValue %[[#PTR_TO_U32]]
+define spir_func i32 @ptrtoint_very_short() {
+ ret i32 ptrtoint(
+ ptr addrspace(7) getelementptr(
+ i32, ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)),
+ i32 3) to i32)
+}
+
+; CHECK: %[[#PTRTOINT_VEC]] = OpFunction %[[#VEC2_I160_TY]]
+; CHECK: OpReturnValue %[[#NULL_VEC2_I160]]
+define spir_func <2 x i160> @ptrtoint_vec() {
+ ret <2 x i160> ptrtoint (<2 x ptr addrspace(7)> zeroinitializer to <2 x i160>)
+}
+
+; CHECK: %[[#INTTOPTR]] = OpFunction %[[#I8PTR_ADDRSPACE_7]]
+; CHECK: OpReturnValue %[[#NULLPTR_I8PTR_ADDRSPACE_7]]
+define spir_func ptr addrspace(7) @inttoptr() {
+ ret ptr addrspace(7) inttoptr (i160 0 to ptr addrspace(7))
+}
+
+; CHECK: %[[#INTTOPTR_VEC]] = OpFunction %[[#VEC2_I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#TMPV:]] = OpCompositeInsert %[[#VEC2_I8PTR_ADDRSPACE_7]] %[[#CONSTEXPR_CAST_INT_1_TO_PTR_AS7]] %[[#]] 0
+; CHECK: %[[#V:]] = OpCompositeInsert %[[#VEC2_I8PTR_ADDRSPACE_7]] %[[#CONSTEXPR_CAST_INT_2_TO_PTR_AS7]] %[[#TMPV]] 1
+; OpReturnValue %[[#V]]
+define spir_func <2 x ptr addrspace(7)> @inttoptr_vec() {
+ ret <2 x ptr addrspace(7)> inttoptr (<2 x i160> <i160 1, i160 2> to <2 x ptr addrspace(7)>)
+}
+
+; CHECK: %[[#FANCY_ZERO]] = OpFunction %[[#INT32_TY]]
+; CHECK: %[[#BUF_AS8_TO_AS7_5:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST_P8_P7]] %[[#BUF]]
+; CHECK: %[[#PTR_TO_U32_1:]] = OpConvertPtrToU %[[#INT32_TY]] %[[#BUF_AS8_TO_AS7_5]]
+; CHECK: OpReturnValue %[[#PTR_TO_U32_1]]
+define spir_func i32 @fancy_zero() {
+ ret i32 ptrtoint (
+ ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7))
+ to i32)
+}
+
+; CHECK: %[[#LOAD_NULL]] = OpFunction %[[#INT32_TY]]
+; CHECK: %[[#BC8:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#NULLPTR_I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#LD:]] = OpLoad %[[#INT32_TY]] %[[#BC8]]
+; CHECK: OpReturnValue %[[#LD]]
+define spir_func i32 @load_null() {
+ %x = load i32, ptr addrspace(7) null, align 4
+ ret i32 %x
+}
+
+; CHECK: %[[#STORE_NULL]] = OpFunction
+; CHECK: %[[#BC9:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#NULLPTR_I8PTR_ADDRSPACE_7]]
+; CHECK: OpStore %[[#BC9]]
+define spir_func void @store_null() {
+ store i32 0, ptr addrspace(7) null, align 4
+ ret void
+}
+
+; CHECK: %[[#LOAD_POISON]] = OpFunction %[[#INT32_TY]]
+; CHECK: %[[#BC10:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#UNDEF_I8PTR_ADDRSPACE_7:]]
+; CHECK: %[[#LD1:]] = OpLoad %[[#INT32_TY]] %[[#BC10]]
+; CHECK: OpReturnValue %[[#LD1]]
+define spir_func i32 @load_poison() {
+ %x = load i32, ptr addrspace(7) poison, align 4
+ ret i32 %x
+}
+
+; CHECK: %[[#STORE_POISON]] = OpFunction
+; CHECK: %[[#BC11:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#UNDEF_I8PTR_ADDRSPACE_7:]]
+; CHECK: OpStore %[[#BC11]]
+define spir_func void @store_poison() {
+ store i32 0, ptr addrspace(7) poison, align 4
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-dead-intrinsics.ll b/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-dead-intrinsics.ll
new file mode 100644
index 0000000000000..6b8229bcc7b1e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-dead-intrinsics.ll
@@ -0,0 +1,9 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+target triple = "spirv64-amd-amdhsa"
+
+declare spir_kernel void @arbitrary(ptr addrspace(1))
+
+declare spir_func <4 x i8> @llvm.masked.load.v4i8.p7(ptr addrspace(7) captures(none), i32 immarg, <4 x i1>, <4 x i8>)
+; CHECK-NOT: llvm_masked_load
diff --git a/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-memops.ll b/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-memops.ll
new file mode 100644
index 0000000000000..3a1017538355d
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-memops.ll
@@ -0,0 +1,162 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; spirv-val incorrectly flags atomic accesses as not allowed under universal validation for DeviceOnlyINTEL/HostOnlyINTEL
+; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+target triple = "spirv64-amd-amdhsa"
+
+; CHECK: OpName %[[#LOADS:]] "loads"
+; CHECK: OpName %[[#OPAQUE_PTR_CAST:]] "spirv.llvm_spv_opaque_ptr_cast_p7_p8"
+; CHECK: OpName %[[#STORES:]] "stores"
+; CHECK: OpName %[[#ATOMICRMW:]] "atomicrmw"
+; CHECK: OpName %[[#CMPXCHG:]] "cmpxchg"
+
+; CHECK: %[[#LOADS]] = OpFunction
+; CHECK: %[[#BUF:]] = OpFunctionParameter %[[#I8PTR_ADDRSPACE_8:]]
+; CHECK: %[[#CAST_AS8_TO_AS7:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7:]] %[[#OPAQUE_PTR_CAST]] %[[#BUF]]
+; CHECK: %[[#BC:]] = OpBitcast %[[#FLOATPTR_ADDRSPACE_7:]] %[[#CAST_AS8_TO_AS7]]
+; CHECK: %[[#P:]] = OpPtrAccessChain %[[#FLOATPTR_ADDRSPACE_7:]] %[[#BC]]
+; CHECK: %[[#SCALAR:]] = OpLoad %[[#FLOAT_TY:]] %[[#P]] Volatile|Aligned 4
+; CHECK: %[[#P_AS_VEC2:]] = OpBitcast %[[#VEC2_FLOATPTR_ADDRSPACE_7:]] %[[#P]]
+; CHECK: %[[#VEC2:]] = OpLoad %[[#VEC2_FLOAT_TY:]] %[[#P_AS_VEC2]] Volatile|Aligned 8
+; CHECK: %[[#P_AS_VEC4:]] = OpBitcast %[[#VEC4_FLOATPTR_ADDRSPACE_7:]] %[[#P]]
+; CHECK: %[[#VEC4:]] = OpLoad %[[#VEC4_FLOAT_TY:]] %[[#P_AS_VEC4]] Volatile|Aligned 16
+; CHECK: %[[#NONTEMPORAL:]] = OpLoad %[[#FLOAT_TY]] %[[#P]] Volatile|Aligned|Nontemporal 4
+; CHECK: %[[#P_AS_INT32:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7:]] %[[#P]]
+; CHECK: %[[#ATOMIC:]] = OpAtomicLoad %[[#INT32_TY:]] %[[#P_AS_INT32]]
+; CHECK: %[[#ATOMIC_MONOTONIC:]] = OpAtomicLoad %[[#INT32_TY]] %[[#P_AS_INT32]]
+; CHECK: %[[#ATOMIC_ACQUIRE:]] = OpAtomicLoad %[[#INT32_TY]] %[[#P_AS_INT32]]
+define void @loads(ptr addrspace(8) %buf) {
+ %base = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %p = getelementptr float, ptr addrspace(7) %base, i32 4
+
+ %scalar = load volatile float, ptr addrspace(7) %p, align 4
+ %vec2 = load volatile <2 x float>, ptr addrspace(7) %p, align 8
+ %vec4 = load volatile <4 x float>, ptr addrspace(7) %p, align 16
+
+ %nontemporal = load volatile float, ptr addrspace(7) %p, !nontemporal !0
+
+ %atomic = load atomic volatile float, ptr addrspace(7) %p syncscope("subgroup") seq_cst, align 4
+ %atomic.monotonic = load atomic float, ptr addrspace(7) %p syncscope("subgroup") monotonic, align 4
+ %atomic.acquire = load atomic float, ptr addrspace(7) %p acquire, align 4
+
+ ret void
+}
+
+; CHECK: %[[#STORES]] = OpFunction
+; CHECK: %[[#BUF1:]] = OpFunctionParameter %[[#I8PTR_ADDRSPACE_8]]
+; CHECK: %[[#F:]] = OpFunctionParameter %[[#FLOAT_TY]]
+; CHECK: %[[#F4:]] = OpFunctionParameter %[[#VEC4_FLOAT_TY]]
+; CHECK: %[[#CAST_AS8_TO_AS7_1:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST]] %[[#BUF1]]
+; CHECK: %[[#BC1:]] = OpBitcast %[[#FLOATPTR_ADDRSPACE_7]] %[[#CAST_AS8_TO_AS7_1]]
+; CHECK: %[[#P1:]] = OpPtrAccessChain %[[#FLOATPTR_ADDRSPACE_7]] %[[#BC1]]
+; CHECK: OpStore %[[#P1]] %[[#F]] Aligned 4
+; CHECK: %[[#P1_AS_VEC4:]] = OpBitcast %[[#VEC4_FLOATPTR_ADDRSPACE_7]] %[[#P1]]
+; CHECK: OpStore %[[#P1_AS_VEC4]] %[[#F4]] Aligned 16
+; CHECK: OpStore %[[#P1]] %[[#F]] Aligned|Nontemporal 4
+; CHECK: OpStore %[[#P1]] %[[#F]] Volatile|Aligned 4
+; CHECK: OpStore %[[#P1]] %[[#F]] Volatile|Aligned|Nontemporal 4
+; CHECK: %[[#F_AS_INT32:]] = OpBitcast %[[#INT32_TY]] %[[#F]]
+; CHECK: %[[#P1_AS_INT32:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#P1]]
+; CHECK: OpAtomicStore %[[#P1_AS_INT32]] %[[#]] %[[#]] %[[#F_AS_INT32]]
+; CHECK: %[[#F_AS_INT32_1:]] = OpBitcast %[[#INT32_TY]] %[[#F]]
+; OpAtomicStore %[[#P1_AS_INT32_1]] %[[#]] %[[#]] %[[#F_AS_INT32_1]]
+; CHECK: %[[#F_AS_INT32_2:]] = OpBitcast %[[#INT32_TY]] %[[#F]]
+; OpAtomicStore %[[#P1_AS_INT32_2]] %[[#]] %[[#]] %[[#F_AS_INT32_2]]
+define void @stores(ptr addrspace(8) %buf, float %f, <4 x float> %f4) {
+ %base = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %p = getelementptr float, ptr addrspace(7) %base, i32 4
+
+ store float %f, ptr addrspace(7) %p, align 4
+ store <4 x float> %f4, ptr addrspace(7) %p, align 16
+
+ store float %f, ptr addrspace(7) %p, !nontemporal !0
+
+ store volatile float %f, ptr addrspace(7) %p
+ store volatile float %f, ptr addrspace(7) %p, !nontemporal !0
+
+ store atomic volatile float %f, ptr addrspace(7) %p syncscope("subgroup") seq_cst, align 4
+ store atomic float %f, ptr addrspace(7) %p syncscope("subgroup") monotonic, align 4
+ store atomic float %f, ptr addrspace(7) %p release, align 4
+
+ ret void
+}
+
+; CHECK: %[[#ATOMICRMW]] = OpFunction
+; CHECK: %[[#BUF2:]] = OpFunctionParameter %[[#I8PTR_ADDRSPACE_8]]
+; CHECK: %[[#F1:]] = OpFunctionParameter %[[#FLOAT_TY]]
+; CHECK: %[[#I:]] = OpFunctionParameter %[[#INT32_TY]]
+; CHECK: %[[#CAST_AS8_TO_AS7_2:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST]] %[[#BUF2]]
+; CHECK: %[[#BC2:]] = OpBitcast %[[#FLOATPTR_ADDRSPACE_7]] %[[#CAST_AS8_TO_AS7_2]]
+; CHECK: %[[#P2:]] = OpPtrAccessChain %[[#FLOATPTR_ADDRSPACE_7]] %[[#BC2]]
+; CHECK: %[[#P2_AS_INT32:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#P2]]
+; CHECK: %[[#P2_AS_INT32_1:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#P2]]
+; CHECK: %[[#P2_AS_INT32_2:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#P2]]
+; CHECK: %[[#P2_AS_INT32_3:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#P2]]
+; CHECK: %[[#P2_AS_INT32_4:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#P2]]
+; CHECK: %[[#P2_AS_INT32_5:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#P2]]
+; CHECK: %[[#P2_AS_INT32_6:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#P2]]
+; CHECK: %[[#P2_AS_INT32_7:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#P2]]
+; CHECK: %[[#P2_AS_INT32_8:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#P2]]
+; CHECK: %[[#P2_AS_INT32_9:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#P2]]
+; CHECK: %[[#P2_AS_INT32_10:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#P2]]
+; CHECK: %78 = OpAtomicExchange %[[#INT32_TY]] %[[#P2_AS_INT32]] %[[#]] %[[#]] %[[#I]]
+; CHECK: %79 = OpAtomicIAdd %[[#INT32_TY]] %[[#P2_AS_INT32_1]] %[[#]] %[[#]] %[[#I]]
+; CHECK: %80 = OpAtomicISub %[[#INT32_TY]] %[[#P2_AS_INT32_2]] %[[#]] %[[#]] %[[#I]]
+; CHECK: %81 = OpAtomicAnd %[[#INT32_TY]] %[[#P2_AS_INT32_3]] %[[#]] %[[#]] %[[#I]]
+; CHECK: %82 = OpAtomicOr %[[#INT32_TY]] %[[#P2_AS_INT32_4]] %[[#]] %[[#]] %[[#I]]
+; CHECK: %83 = OpAtomicXor %[[#INT32_TY]] %[[#P2_AS_INT32_5]] %[[#]] %[[#]] %[[#I]]
+; CHECK: %84 = OpAtomicSMin %[[#INT32_TY]] %[[#P2_AS_INT32_6]] %[[#]] %[[#]] %[[#I]]
+; CHECK: %85 = OpAtomicSMax %[[#INT32_TY]] %[[#P2_AS_INT32_7]] %[[#]] %[[#]] %[[#I]]
+; CHECK: %86 = OpAtomicUMin %[[#INT32_TY]] %[[#P2_AS_INT32_8]] %[[#]] %[[#]] %[[#I]]
+; CHECK: %87 = OpAtomicUMax %[[#INT32_TY]] %[[#P2_AS_INT32_9]] %[[#]] %[[#]] %[[#I]]
+; CHECK: %88 = OpAtomicFAddEXT %[[#FLOAT_TY]] %[[#P2]] %23 %22 %[[#F1]]
+; CHECK: %89 = OpAtomicFMaxEXT %[[#FLOAT_TY]] %[[#P2]] %23 %22 %[[#F1]]
+; CHECK: %90 = OpAtomicFMinEXT %[[#FLOAT_TY]] %[[#P2]] %23 %22 %[[#F1]]
+; CHECK: %91 = OpAtomicIAdd %[[#INT32_TY]] %[[#P2_AS_INT32_10]] %[[#]] %[[#]] %[[#I]]
+define void @atomicrmw(ptr addrspace(8) %buf, float %f, i32 %i) {
+ %base = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %p = getelementptr float, ptr addrspace(7) %base, i32 4
+
+ ; Fence insertion is tested by loads and stores
+ %xchg = atomicrmw xchg ptr addrspace(7) %p, i32 %i syncscope("subgroup") seq_cst, align 4
+ %add = atomicrmw add ptr addrspace(7) %p, i32 %i syncscope("subgroup") seq_cst, align 4
+ %sub = atomicrmw sub ptr addrspace(7) %p, i32 %i syncscope("subgroup") seq_cst, align 4
+ %and = atomicrmw and ptr addrspace(7) %p, i32 %i syncscope("subgroup") seq_cst, align 4
+ %or = atomicrmw or ptr addrspace(7) %p, i32 %i syncscope("subgroup") seq_cst, align 4
+ %xor = atomicrmw xor ptr addrspace(7) %p, i32 %i syncscope("subgroup") seq_cst, align 4
+ %min = atomicrmw min ptr addrspace(7) %p, i32 %i syncscope("subgroup") seq_cst, align 4
+ %max = atomicrmw max ptr addrspace(7) %p, i32 %i syncscope("subgroup") seq_cst, align 4
+ %umin = atomicrmw umin ptr addrspace(7) %p, i32 %i syncscope("subgroup") seq_cst, align 4
+ %umax = atomicrmw umax ptr addrspace(7) %p, i32 %i syncscope("subgroup") seq_cst, align 4
+
+ %fadd = atomicrmw fadd ptr addrspace(7) %p, float %f syncscope("subgroup") seq_cst, align 4
+ %fmax = atomicrmw fmax ptr addrspace(7) %p, float %f syncscope("subgroup") seq_cst, align 4
+ %fmin = atomicrmw fmin ptr addrspace(7) %p, float %f syncscope("subgroup") seq_cst, align 4
+
+ ; Check a no-return atomic
+ atomicrmw add ptr addrspace(7) %p, i32 %i syncscope("subgroup") seq_cst, align 4
+
+ ret void
+}
+
+; CHECK: %[[#CMPXCHG]] = OpFunction
+; CHECK: %[[#BUF3:]] = OpFunctionParameter %[[#I8PTR_ADDRSPACE_8]]
+; CHECK: %[[#WANTED:]] = OpFunctionParameter %[[#INT32_TY]]
+; CHECK: %[[#NEW:]] = OpFunctionParameter %[[#INT32_TY]]
+; CHECK: %[[#CAST_AS8_TO_AS7_3:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST]] %[[#BUF3]]
+; CHECK: %[[#BC3:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#CAST_AS8_TO_AS7_3]]
+; CHECK: %[[#P3:]] = OpPtrAccessChain %[[#I32PTR_ADDRSPACE_7]] %[[#BC3]]
+; CHECK: %[[#P3_AS_STRUCT:]] = OpBitcast %[[#STRUCTPTR_ADDRSPACE_7:]] %[[#P3]]
+; CHECK: %[[#P3_AS_INT32:]] = OpBitcast %[[#I32PTR_ADDRSPACE_7]] %[[#P3_AS_STRUCT]]
+; CHECK: %[[#CMPX:]] = OpAtomicCompareExchange %[[#INT32_TY]] %[[#P3_AS_INT32]] %[[#]] %[[#]] %[[#]] %[[#NEW]] %[[#WANTED]]
+; CHECK: %[[#SUCCESS:]] = OpIEqual %[[#BOOL_TY:]] %[[#CMPX]] %[[#WANTED]]
+define {i32, i1} @cmpxchg(ptr addrspace(8) %buf, i32 %wanted, i32 %new) {
+ %base = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %p = getelementptr i32, ptr addrspace(7) %base, i32 4
+
+ %ret = cmpxchg volatile ptr addrspace(7) %p, i32 %wanted, i32 %new syncscope("subgroup") acq_rel monotonic, align 4
+ ret {i32, i1} %ret
+}
+
+!0 = ! { i32 1 }
+!1 = ! { }
diff --git a/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-p7-in-memory.ll b/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-p7-in-memory.ll
new file mode 100644
index 0000000000000..2388f9918f33e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-p7-in-memory.ll
@@ -0,0 +1,75 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+target triple = "spirv64-amd-amdhsa"
+
+; CHECK: OpName %[[#SCALAR_COPY:]] "scalar_copy"
+; CHECK: OpName %[[#VECTOR_COPY:]] "vector_copy"
+; CHECK: OpName %[[#ALLOCA:]] "alloca"
+; CHECK: OpName %[[#COMPLEX_COPY:]] "complex_copy"
+; CHECK: %[[#INT8_TY:]] = OpTypeInt 8 0
+; CHECK: %[[#I8PTR_ADDRSPACE_7:]] = OpTypePointer DeviceOnlyINTEL %[[#INT8_TY]]
+; CHECK: %[[#I8PTR_ADDRSPACE_7PTR_GENERIC:]] = OpTypePointer Generic %[[#I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#VEC4_I8PTR_ADDRSPACE_7:]] = OpTypeVector %[[#I8PTR_ADDRSPACE_7]] 4
+; CHECK: %[[#VEC4_I8PTR_ADDRSPACE_7PTR_GENERIC:]] = OpTypePointer Generic %[[#VEC4_I8PTR_ADDRSPACE_7]]
+; CHECK: %[[#I8PTR_ADDRSPACE_7PTR_PRIVATE:]] = OpTypePointer Function %[[#I8PTR_ADDRSPACE_7]]
+
+; CHECK: %[[#SCALAR_COPY]] = OpFunction
+; CHECK: %[[#SCALAR_COPY_A:]] = OpFunctionParameter %[[#I8PTR_ADDRSPACE_7PTR_GENERIC]]
+; CHECK: %[[#SCALAR_COPY_B:]] = OpFunctionParameter %[[#I8PTR_ADDRSPACE_7PTR_GENERIC]]
+; CHECK: %[[#SCALAR_COPY_X:]] = OpLoad %[[#I8PTR_ADDRSPACE_7]] %[[#SCALAR_COPY_A]]
+; CHECK: %[[#SCALAR_COPY_B1:]] = OpPtrAccessChain %[[#I8PTR_ADDRSPACE_7PTR_GENERIC]] %[[#SCALAR_COPY_B]]
+; CHECK: OpStore %[[#SCALAR_COPY_B1]] %[[#SCALAR_COPY_X]]
+define void @scalar_copy(ptr addrspace(4) %a, ptr addrspace(4) %b) {
+ %x = load ptr addrspace(7), ptr addrspace(4) %a
+ %b1 = getelementptr ptr addrspace(7), ptr addrspace(4) %b, i64 1
+ store ptr addrspace(7) %x, ptr addrspace(4) %b1
+ ret void
+}
+
+; CHECK: %[[#VECTOR_COPY:]] = OpFunction
+; CHECK: %[[#VECTOR_COPY_A:]] = OpFunctionParameter %[[#VEC4_I8PTR_ADDRSPACE_7PTR_GENERIC]]
+; CHECK: %[[#VECTOR_COPY_B:]] = OpFunctionParameter %[[#VEC4_I8PTR_ADDRSPACE_7PTR_GENERIC]]
+; CHECK: %[[#VECTOR_COPY_X:]] = OpLoad %[[#VEC4_I8PTR_ADDRSPACE_7]] %[[#VECTOR_COPY_A]]
+; CHECK: %[[#VECTOR_COPY_B1:]] = OpPtrAccessChain %[[#VEC4_I8PTR_ADDRSPACE_7PTR_GENERIC]] %[[#VECTOR_COPY_B]]
+; CHECK: %[[#VECTOR_COPY_B1_BC:]] = OpBitcast %[[#VEC4PTR_VEC4_I8PTR_ADDRSPACE_7:]] %[[#VECTOR_COPY_B1]]
+; CHECK: %[[#VECTOR_COPY_B1_BC_BC:]] = OpBitcast %[[#VEC4_I8PTR_ADDRSPACE_7PTR_GENERIC]] %[[#VECTOR_COPY_B1_BC]]
+; CHECK: OpStore %[[#VECTOR_COPY_B1_BC_BC]] %[[#VECTOR_COPY_X]]
+define void @vector_copy(ptr addrspace(4) %a, ptr addrspace(4) %b) {
+ %x = load <4 x ptr addrspace(7)>, ptr addrspace(4) %a
+ %b1 = getelementptr <4 x ptr addrspace(7)>, ptr addrspace(4) %b, i64 2
+ store <4 x ptr addrspace(7)> %x, ptr addrspace(4) %b1
+ ret void
+}
+
+; CHECK: %[[#ALLOCA]] = OpFunction
+; CHECK: %[[#ALLOCA_A:]] = OpFunctionParameter %[[#I8PTR_ADDRSPACE_7PTR_GENERIC]]
+; CHECK: %[[#ALLOCA_B:]] = OpFunctionParameter %[[#I8PTR_ADDRSPACE_7PTR_GENERIC]]
+; CHECK: %[[#ALLOCA_ARR:]] = OpVariable %[[#ARR5_I8PTR_ADDRSPACE_7:]] Function
+; CHECK: %[[#ALLOCA_X:]] = OpLoad %[[#I8PTR_ADDRSPACE_7]] %[[#ALLOCA_A]]
+; CHECK: %[[#ALLOCA_ARR_BC:]] = OpBitcast %[[#I8PTR_ADDRSPACE_7PTR_PRIVATE]] %[[#ALLOCA_ARR]]
+; CHECK: %[[#ALLOCA_L:]] = OpPtrAccessChain %[[#I8PTR_ADDRSPACE_7PTR_PRIVATE]] %[[#ALLOCA_ARR_BC]] %[[#]]
+; CHECK: OpStore %[[#ALLOCA_L]] %[[#ALLOCA_X]]
+; CHECK: %[[#ALLOCA_Y:]] = OpLoad %[[#I8PTR_ADDRSPACE_7]] %[[#ALLOCA_L]]
+; CHECK: OpStore %[[#ALLOCA_B]] %[[#ALLOCA_Y]]
+define void @alloca(ptr addrspace(4) %a, ptr addrspace(4) %b) {
+ %alloca = alloca [5 x ptr addrspace(7)]
+ %x = load ptr addrspace(7), ptr addrspace(4) %a
+ %l = getelementptr ptr addrspace(7), ptr %alloca, i32 1
+ store ptr addrspace(7) %x, ptr %l
+ %y = load ptr addrspace(7), ptr %l
+ store ptr addrspace(7) %y, ptr addrspace(4) %b
+ ret void
+}
+
+; CHECK: %[[#COMPLEX_COPY]] = OpFunction
+; CHECK: %[[#COMPLEX_COPY_A:]] = OpFunctionParameter %[[#STRUCT1_PTR_TY:]]
+; CHECK: %[[#COMPLEX_COPY_B:]] = OpFunctionParameter %[[#WRAPPED_STRUCT1_PTR_TY:]]
+; CHECK: %[[#COMPLEX_COPY_X:]] = OpLoad %[[#STRUCT1_TY:]] %[[#COMPLEX_COPY_A]]
+; CHECK: %[[#COMPLEX_COPY_B_BC:]] = OpBitcast %[[#STRUCT1_PTR_TY]] %[[#COMPLEX_COPY_B]]
+; CHECK: OpStore %[[#COMPLEX_COPY_B_BC]] %[[#COMPLEX_COPY_X]]
+define void @complex_copy(ptr addrspace(4) %a, ptr addrspace(4) %b) {
+ %x = load {[2 x ptr addrspace(7)], i32, ptr addrspace(7)}, ptr addrspace(4) %a
+ store {[2 x ptr addrspace(7)], i32, ptr addrspace(7)} %x, ptr addrspace(4) %b
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-unoptimized-debug-data.ll b/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-unoptimized-debug-data.ll
new file mode 100644
index 0000000000000..236c4a1fc9cd7
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/amdgcnspirv-lower-buffer-fat-pointers-unoptimized-debug-data.ll
@@ -0,0 +1,142 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+target triple = "spirv64-amd-amdhsa"
+
+; CHECK: OpCapability USMStorageClassesINTEL
+; CHECK: OpExtension "SPV_INTEL_usm_storage_classes"
+; CHECK: OpExtension "SPV_KHR_non_semantic_info"
+; CHECK: OpName %[[#DEBUG_STASH_POINTER:]] "debug_stash_pointer"
+; CHECK: OpName %[[#OPAQUE_PTR_CAST_P7_P8:]] "spirv.llvm_spv_opaque_ptr_cast_p7_p8"
+; CHECK: %[[#INT8_TY:]] = OpTypeInt 8 0
+; CHECK: %3 = OpTypePointer HostOnlyINTEL %2
+ ; %4 = OpTypeInt 32 0
+ ; %5 = OpTypeFloat 32
+ ; %6 = OpTypeFunction %5 %3 %4 %3
+ ; %7 = OpTypePointer DeviceOnlyINTEL %2
+ ; %8 = OpTypeFunction %7 %3
+ ; %9 = OpTypePointer Function %7
+ ; %10 = OpTypePointer DeviceOnlyINTEL %5
+ ; %11 = OpTypePointer Function %10
+ ; %12 = OpTypePointer DeviceOnlyINTEL %10
+ ; %13 = OpTypePointer CrossWorkgroup %4
+ ; %14 = OpConstantNull %4
+ ; %15 = OpTypePointer Function %12
+ ; %16 = OpVariable %13 CrossWorkgroup %14
+ ; %41 = OpTypeVoid
+ ; %42 = OpConstant %4 100
+ ; %43 = OpConstant %4 0
+ ; %44 = OpExtInst %41 %37 DebugSource %38
+ ; %45 = OpExtInst %41 %37 DebugCompilationUnit %42 %43 %44 %43
+ ; %46 = OpConstant %4 32
+ ; %47 = OpConstant %4 6
+ ; %48 = OpExtInst %41 %37 DebugTypeBasic %39 %46 %47 %43
+ ; %49 = OpConstant %4 256
+ ; %50 = OpExtInst %41 %37 DebugTypeBasic %40 %49 %47 %43
+ ; %17 = OpFunction %7 None %8
+ ; %18 = OpFunctionParameter %3
+ ; OpFunctionEnd
+
+; CHECK: %[[#DEBUG_STASH_POINTER]] = OpFunction
+; CHECK: %[[#BUF:]] = OpFunctionParameter %[[#I8PTR_ADDRSPACE_8:]]
+; CHECK: %[[#IDX:]] = OpFunctionParameter %[[#INT32_TY:]]
+; CHECK: %[[#AUX:]] = OpFunctionParameter %[[#I8PTR_ADDRSPACE_8]]
+; CHECK: %[[#BUF_PTR_VAR:]] = OpVariable %[[#PTRI8PTR_ADDRSPACE_7_PRIVATE:]] Function
+; CHECK: %[[#AUX_PTR_VAR:]] = OpVariable %[[#PTRI8PTR_ADDRSPACE_7_PRIVATE:]] Function
+; CHECK: DEBUG_VALUE: debug_stash_pointer:1 <- %11
+; CHECK: DEBUG_VALUE: debug_stash_pointer:2 <- %14
+; CHECK: %[[#BUF_TO_AS7:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7:]] %[[#OPAQUE_PTR_CAST_P7_P8]] %[[#BUF]]
+; CHECK: DEBUG_VALUE: debug_stash_pointer:3 <- %16
+; CHECK: OpStore %[[#BUF_PTR_VAR]] %[[#BUF_TO_AS7]]
+; CHECK: %[[#AUX_TO_AS7:]] = OpFunctionCall %[[#I8PTR_ADDRSPACE_7]] %[[#OPAQUE_PTR_CAST_P7_P8]] %[[#AUX]]
+; CHECK: DEBUG_VALUE: debug_stash_pointer:4 <- %21
+; CHECK: OpStore %[[#AUX_PTR_VAR]] %[[#AUX_TO_AS7]]
+; CHECK: %[[#BC_BUF_PTR_VAR_TO_PTRI32PTR_ADDRSPACE_7_PRIVATE:]] = OpBitcast %[[#PTRI32PTR_ADDRSPACE_7_PRIVATE:]] %[[#BUF_PTR_VAR]]
+; CHECK: %[[#BUF_PTR_2:]] = OpLoad %[[#I32PTR_ADDRSPACE_7:]] %[[#BC_BUF_PTR_VAR_TO_PTRI32PTR_ADDRSPACE_7_PRIVATE]]
+; CHECK: DEBUG_VALUE: debug_stash_pointer:5 <- %22
+; CHECK: %[[#BUF_PTR_3:]] = OpPtrAccessChain %[[#I32PTR_ADDRSPACE_7]] %[[#BUF_PTR_2]] %[[#IDX]]
+; CHECK: DEBUG_VALUE: debug_stash_pointer:6 <- %23
+; CHECK: %[[#BC_BUF_PTR_VAR_TO_PTRI32PTR_ADDRSPACE_7_PRIVATE_1:]] = OpBitcast %[[#PTRI32PTR_ADDRSPACE_7_PRIVATE]] %[[#BUF_PTR_VAR]]
+; CHECK: OpStore %[[#BC_BUF_PTR_VAR_TO_PTRI32PTR_ADDRSPACE_7_PRIVATE_1]] %[[#BUF_PTR_3]]
+; CHECK: %[[#BC_BUF_PTR_VAR_TO_PTRI32PTR_ADDRSPACE_7_PRIVATE_2:]] = OpBitcast %[[#PTRI32PTR_ADDRSPACE_7_PRIVATE]] %[[#BUF_PTR_VAR]]
+; CHECK: %[[#BUF_PTR_4:]] = OpLoad %[[#PTRFLOATPTR_ADDRSPACE_7_PRIVATE:]] %[[#BC_BUF_PTR_VAR_TO_PTRI32PTR_ADDRSPACE_7_PRIVATE_2]]
+; CHECK: DEBUG_VALUE: debug_stash_pointer:7 <- %25
+; CHECK: %[[#RET:]] = OpLoad %[[#FLOAT:]] %[[#BUF_PTR_4]]
+; CHECK: DEBUG_VALUE: debug_stash_pointer:8 <- %26
+; CHECK: %[[#BC_AUX_PTR_VAR_TO_PTRI32PTR_ADDRSPACE_7_PRIVATE_3:]] = OpBitcast %[[#PTRPTRI32PTR_ADDRSPACE_7_PRIVATE_PRIVATE:]] %[[#AUX_PTR_VAR]]
+; CHECK: %[[#AUX_PTR_2:]] = OpLoad %[[#]] %[[#BC_AUX_PTR_VAR_TO_PTRI32PTR_ADDRSPACE_7_PRIVATE_3]]
+; CHECK: DEBUG_VALUE: debug_stash_pointer:9 <- %27
+; CHECK: OpStore %[[#AUX_PTR_2]] %[[#BUF_PTR_4]]
+; CHECK: OpReturnValue %[[#RET]]
+
+define float @debug_stash_pointer(ptr addrspace(8) %buf, i32 %idx, ptr addrspace(8) %aux) !dbg !5 {
+ %buf.ptr.var = alloca ptr addrspace(7), align 32, !dbg !20
+ call void @llvm.dbg.value(metadata ptr %buf.ptr.var, metadata !9, metadata !DIExpression()), !dbg !20
+ %aux.ptr.var = alloca ptr addrspace(7), align 32, !dbg !21
+ call void @llvm.dbg.value(metadata ptr %aux.ptr.var, metadata !11, metadata !DIExpression()), !dbg !21
+ %buf.ptr = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7), !dbg !22
+ call void @llvm.dbg.value(metadata ptr addrspace(7) %buf.ptr, metadata !12, metadata !DIExpression()), !dbg !22
+ store ptr addrspace(7) %buf.ptr, ptr %buf.ptr.var, align 32, !dbg !23, !DIAssignID !40
+ call void @llvm.dbg.assign(metadata ptr addrspace(7) %buf.ptr, metadata !12, metadata !DIExpression(), metadata !40, metadata ptr %buf.ptr.var, metadata !DIExpression()), !dbg !20
+ %aux.ptr = addrspacecast ptr addrspace(8) %aux to ptr addrspace(7), !dbg !24
+ call void @llvm.dbg.value(metadata ptr addrspace(7) %aux.ptr, metadata !14, metadata !DIExpression()), !dbg !24
+ store ptr addrspace(7) %aux.ptr, ptr %aux.ptr.var, align 32, !dbg !25
+ %buf.ptr.2 = load ptr addrspace(7), ptr %buf.ptr.var, align 32, !dbg !26
+ call void @llvm.dbg.value(metadata ptr addrspace(7) %buf.ptr.2, metadata !15, metadata !DIExpression()), !dbg !26
+ %buf.ptr.3 = getelementptr float, ptr addrspace(7) %buf.ptr.2, i32 %idx, !dbg !27
+ call void @llvm.dbg.value(metadata ptr addrspace(7) %buf.ptr.3, metadata !16, metadata !DIExpression()), !dbg !27
+ store ptr addrspace(7) %buf.ptr.3, ptr %buf.ptr.var, align 32, !dbg !28
+ %buf.ptr.4 = load ptr addrspace(7), ptr %buf.ptr.var, align 32, !dbg !29
+ call void @llvm.dbg.value(metadata ptr addrspace(7) %buf.ptr.4, metadata !17, metadata !DIExpression()), !dbg !29
+ %ret = load float, ptr addrspace(7) %buf.ptr.4, align 4, !dbg !30
+ call void @llvm.dbg.value(metadata float %ret, metadata !18, metadata !DIExpression()), !dbg !30
+ %aux.ptr.2 = load ptr addrspace(7), ptr %aux.ptr.var, align 32, !dbg !31
+ call void @llvm.dbg.value(metadata ptr addrspace(7) %aux.ptr.2, metadata !19, metadata !DIExpression()), !dbg !31
+ store ptr addrspace(7) %buf.ptr.4, ptr addrspace(7) %aux.ptr.2, align 32, !dbg !32
+ ret float %ret, !dbg !33
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+
+attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!2, !3}
+!llvm.module.flags = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "<stdin>", directory: "/")
+!2 = !{i32 14}
+!3 = !{i32 9}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "debug_stash_pointer", linkageName: "debug_stash_pointer", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !{!9, !11, !12, !14, !15, !16, !17, !18, !19}
+!9 = !DILocalVariable(name: "1", scope: !5, file: !1, line: 1, type: !10)
+!10 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned)
+!11 = !DILocalVariable(name: "2", scope: !5, file: !1, line: 2, type: !10)
+!12 = !DILocalVariable(name: "3", scope: !5, file: !1, line: 3, type: !13)
+!13 = !DIBasicType(name: "ty256", size: 256, encoding: DW_ATE_unsigned)
+!14 = !DILocalVariable(name: "4", scope: !5, file: !1, line: 5, type: !13)
+!15 = !DILocalVariable(name: "5", scope: !5, file: !1, line: 7, type: !13)
+!16 = !DILocalVariable(name: "6", scope: !5, file: !1, line: 8, type: !13)
+!17 = !DILocalVariable(name: "7", scope: !5, file: !1, line: 10, type: !13)
+!18 = !DILocalVariable(name: "8", scope: !5, file: !1, line: 11, type: !10)
+!19 = !DILocalVariable(name: "9", scope: !5, file: !1, line: 12, type: !13)
+!20 = !DILocation(line: 1, column: 1, scope: !5)
+!21 = !DILocation(line: 2, column: 1, scope: !5)
+!22 = !DILocation(line: 3, column: 1, scope: !5)
+!23 = !DILocation(line: 4, column: 1, scope: !5)
+!24 = !DILocation(line: 5, column: 1, scope: !5)
+!25 = !DILocation(line: 6, column: 1, scope: !5)
+!26 = !DILocation(line: 7, column: 1, scope: !5)
+!27 = !DILocation(line: 8, column: 1, scope: !5)
+!28 = !DILocation(line: 9, column: 1, scope: !5)
+!29 = !DILocation(line: 10, column: 1, scope: !5)
+!30 = !DILocation(line: 11, column: 1, scope: !5)
+!31 = !DILocation(line: 12, column: 1, scope: !5)
+!32 = !DILocation(line: 13, column: 1, scope: !5)
+!33 = !DILocation(line: 14, column: 1, scope: !5)
+!40 = distinct !DIAssignID()
>From f76bf3f5d8a0c58ac0261377d36e4da01007b817 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu at amd.com>
Date: Sun, 24 May 2026 00:32:37 +0300
Subject: [PATCH 29/29] Tidy up.
---
llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 7 +-
llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp | 11 +--
llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 2 +-
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 6 --
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 14 ++--
llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp | 69 ++++++++++++++-----
.../Target/SPIRV/SPIRVPrepareFunctions.cpp | 2 +-
7 files changed, 71 insertions(+), 40 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 6ef8715783748..8fe7eefc3b66a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -1690,7 +1690,8 @@ static bool genWorkgroupQuery(const SPIRV::IncomingCall *Call,
uint64_t DefaultValue) {
Register IndexRegister = Call->Arguments[0];
const unsigned ResultWidth = Call->ReturnType->getOperand(1).getImm();
- const unsigned PointerSize = GR->getPointerSize();
+ const unsigned PointerSize = GR->getPointerSize(storageClassToAddressSpace(
+ SPIRV::StorageClass::Generic, GR->CurMF->getTarget().getTargetTriple()));
const SPIRVTypeInst PointerSizeType =
GR->getOrCreateSPIRVIntegerType(PointerSize, MIRBuilder);
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
@@ -2828,7 +2829,9 @@ static bool buildNDRange(const SPIRV::IncomingCall *Call,
// Each nd_range field is an array of <Dimension> integers matching the
// address model width (32 or 64 bits).
- const unsigned AddressModelBits = GR->getPointerSize();
+ const unsigned AddressModelBits = GR->getPointerSize(
+ storageClassToAddressSpace(SPIRV::StorageClass::Generic,
+ GR->CurMF->getTarget().getTargetTriple()));
assert(AddressModelBits == 64 || AddressModelBits == 32);
// The dimension is encoded in the function name as "ndrange_XD" where X is
diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index 14c33b5190f5e..44005a0cdccc2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -565,11 +565,12 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
// pointee type.
MRI->setRegClass(ArgReg, SpvType ? GR->getRegClass(SpvType)
: &SPIRV::pIDRegClass);
- MRI->setType(
- ArgReg,
- SpvType ? GR->getRegType(SpvType)
- : LLT::pointer(cast<PointerType>(Arg.Ty)->getAddressSpace(),
- GR->getPointerSize()));
+ if (SpvType) {
+ MRI->setType(ArgReg, GR->getRegType(SpvType));
+ } else {
+ unsigned AS = cast<PointerType>(Arg.Ty)->getAddressSpace();
+ MRI->setType(ArgReg, LLT::pointer(AS, GR->getPointerSize(AS)));
+ }
}
}
if (auto Res = SPIRV::lowerBuiltin(
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index 427aab376d69d..c0e4df66b9b38 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -418,7 +418,7 @@ class SPIRVGlobalRegistry : public SPIRVIRMapping {
getPointerStorageClass(SPIRVTypeInst Type) const;
// Return the number of bits SPIR-V pointers and size_t variables require.
- unsigned getPointerSize(unsigned AS = 0) const {
+ unsigned getPointerSize(unsigned AS) const {
return DL.getPointerSizeInBits(AS);
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index c5499e9d6c1ea..5131a96457600 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -103,12 +103,6 @@ MVT SPIRVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
else if (VT.getVectorElementType() == MVT::i8)
return MVT::v4i8;
}
- // if (STI.getTargetTriple().getVendor() == Triple::VendorType::AMD) {
- // if (VT.isVector() &&
- // VT.getVectorElementType().isExtended())
- // return MVT::v8i32;
- // }
-
return getRegisterType(Context, VT);
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index df03b0cf64bf8..e08e7536a9670 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -6826,11 +6826,10 @@ bool SPIRVInstructionSelector::selectModf(Register ResVReg,
// Create new register for the pointer type of alloca variable.
Register PtrTyReg =
MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
- MIRBuilder.getMRI()->setType(
- PtrTyReg,
- LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function,
- STI.getTargetTriple()),
- GR.getPointerSize()));
+ unsigned AS = storageClassToAddressSpace(SPIRV::StorageClass::Function,
+ STI.getTargetTriple());
+ MIRBuilder.getMRI()->setType(PtrTyReg,
+ LLT::pointer(AS, GR.getPointerSize(AS)));
// Assign SPIR-V type of the pointer type of the alloca variable to the
// new register.
@@ -6946,9 +6945,8 @@ bool SPIRVInstructionSelector::loadBuiltinInputID(
MIRBuilder.getMRI()->createVirtualRegister(GR.getRegClass(PtrType));
unsigned AS = storageClassToAddressSpace(SPIRV::StorageClass::Input,
STI.getTargetTriple());
- MIRBuilder.getMRI()->setType(
- NewRegister,
- LLT::pointer(AS, GR.getPointerSize(AS)));
+ MIRBuilder.getMRI()->setType(NewRegister,
+ LLT::pointer(AS, GR.getPointerSize(AS)));
GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
// Build global variable with the necessary decorations for the input ID
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index bfd937d47b6f1..8b91ab5fbf64a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -84,23 +84,58 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST,
const LLT v2s1 = LLT::fixed_vector(2, 1);
const Triple &TT = TM.getTargetTriple();
- const LLT p0 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function, TT), TM.getPointerSizeInBits(0)); // Function
- const LLT p1 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::CrossWorkgroup, TT), TM.getPointerSizeInBits(1)); // CrossWorkgroup
- const LLT p2 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::UniformConstant, TT), TM.getPointerSizeInBits(2)); // UniformConstant
- const LLT p3 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Workgroup, TT), TM.getPointerSizeInBits(3)); // Workgroup
- const LLT p4 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Generic, TT), TM.getPointerSizeInBits(4)); // Generic
- // Input, SPV_INTEL_usm_storage_classes (Device)
- const LLT p5 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::DeviceOnlyINTEL, TT), TM.getPointerSizeInBits(5));
- // SPV_INTEL_usm_storage_classes (Host)
- const LLT p6 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::HostOnlyINTEL, TT), TM.getPointerSizeInBits(6));
- const LLT p7 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Input, TT), TM.getPointerSizeInBits(7)); // Input
- const LLT p8 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Output, TT), TM.getPointerSizeInBits(8)); // Output
- // CodeSectionINTEL, SPV_INTEL_function_pointers
- const LLT p9 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::CodeSectionINTEL, TT), TM.getPointerSizeInBits(9));
- const LLT p10 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Private, TT), TM.getPointerSizeInBits(10)); // Private
- const LLT p11 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::StorageBuffer, TT), TM.getPointerSizeInBits(11)); // StorageBuffer
- const LLT p12 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Uniform, TT), TM.getPointerSizeInBits(12)); // Uniform
- const LLT p13 = LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::PushConstant, TT), TM.getPointerSizeInBits(13)); // PushConstant
+
+ unsigned FunctionAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::Function, TT);
+ unsigned CrossWorkgroupAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::CrossWorkgroup, TT);
+ unsigned UniformConstantAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::UniformConstant, TT);
+ unsigned WorkgroupAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::Workgroup, TT);
+ unsigned GenericAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::Generic, TT);
+ unsigned DeviceOnlyINTELAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::DeviceOnlyINTEL, TT);
+ unsigned HostOnlyINTELAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::HostOnlyINTEL, TT);
+ unsigned InputAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::Input, TT);
+ unsigned OutputAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::Output, TT);
+ unsigned CodeSectionINTELAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::CodeSectionINTEL, TT);
+ unsigned PrivateAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::Private, TT);
+ unsigned StorageBufferAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::StorageBuffer, TT);
+ unsigned UniformAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::Uniform, TT);
+ unsigned PushConstantAS =
+ storageClassToAddressSpace(SPIRV::StorageClass::PushConstant, TT);
+
+ const LLT p0 = LLT::pointer(FunctionAS, TM.getPointerSizeInBits(FunctionAS));
+ const LLT p1 = LLT::pointer(CrossWorkgroupAS,
+ TM.getPointerSizeInBits(CrossWorkgroupAS));
+ const LLT p2 = LLT::pointer(UniformConstantAS,
+ TM.getPointerSizeInBits(UniformConstantAS));
+ const LLT p3 = LLT::pointer(WorkgroupAS,
+ TM.getPointerSizeInBits(WorkgroupAS));
+ const LLT p4 = LLT::pointer(GenericAS, TM.getPointerSizeInBits(GenericAS));
+ const LLT p5 = LLT::pointer(DeviceOnlyINTELAS,
+ TM.getPointerSizeInBits(DeviceOnlyINTELAS));
+ const LLT p6 = LLT::pointer(HostOnlyINTELAS,
+ TM.getPointerSizeInBits(HostOnlyINTELAS));
+ const LLT p7 = LLT::pointer(InputAS, TM.getPointerSizeInBits(InputAS));
+ const LLT p8 = LLT::pointer(OutputAS, TM.getPointerSizeInBits(OutputAS));
+ const LLT p9 = LLT::pointer(CodeSectionINTELAS,
+ TM.getPointerSizeInBits(CodeSectionINTELAS));
+ const LLT p10 = LLT::pointer(PrivateAS, TM.getPointerSizeInBits(PrivateAS));
+ const LLT p11 = LLT::pointer(StorageBufferAS,
+ TM.getPointerSizeInBits(StorageBufferAS));
+ const LLT p12 = LLT::pointer(UniformAS, TM.getPointerSizeInBits(UniformAS));
+ const LLT p13 = LLT::pointer(PushConstantAS,
+ TM.getPointerSizeInBits(PushConstantAS));
// TODO: remove copy-pasting here by using concatenation in some way.
auto allPtrsScalarsAndVectors = {
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 64503d1d74bdb..c20394d377cb5 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -880,7 +880,7 @@ bool SPIRVPrepareFunctionsImpl::runOnModule(Module &M) {
bool Changed = false;
if (M.functions().empty() ||
all_of(M, [](auto &&F) { return F.isDeclaration(); })) {
- // If there are no functions, insert a service
+ // If there are no defined functions, insert a service
// function so that the global/constant tracking intrinsics
// will be created. Without these intrinsics the generated SPIR-V
// will be empty. The service function itself is not emitted.
More information about the cfe-commits
mailing list