[flang-commits] [clang] [flang] [libc] [libcxx] [lldb] [llvm] [mlir] [GlobalISel] Preserve cache hint metadata through IRTranslator (PR #224522)
Yonah Goldberg via flang-commits
flang-commits at lists.llvm.org
Tue Oct 6 16:49:55 PDT 2026
=?utf-8?q?Mikołaj_Piróg?= <mikolaj.maciej.pirog at intel.com>,Alexander
Shaposhnikov <ashaposhnikov at google.com>,Jonas Devlieghere
<jonas at devlieghere.com>,Nico Weber <thakis at chromium.org>,Vineet Kumar
<173554+vntkmr at users.noreply.github.com>,Aaron Ballman
<aaron at aaronballman.com>,Florian Mayer <fmayer at google.com>,Florian Mayer
<fmayer at google.com>,Alex Duran <alejandro.duran at intel.com>,Jason Van
Beusekom <jason.van-beusekom at hpe.com>,Andy Kaylor <akaylor at nvidia.com>,forking-google-bazel-bot[bot]
=?utf-8?q?,?=Joshua Batista <jbatista at microsoft.com>,Matt Arsenault
<Matthew.Arsenault at amd.com>,Ambrose Leeb <aleeb at nvidia.com>,Henrich Lauko
<xlauko at mail.muni.cz>,Domenic Nutile <domenic.nutile at gmail.com>,Florian Hahn
<flo at fhahn.com>,Julian Nagele <j.nagele at apple.com>,Sam Elliott
<aelliott at qti.qualcomm.com>,Gregory Roth <groth at nvidia.com>,Ben Wu
<soggysocks206 at gmail.com>,Susan Tan =?utf-8?b?KOOCuS3jgrbjg7PjgIA=?=,Grigory
Pastukhov <gpastukhov at meta.com>,Alexey Samsonov <vonosmas at gmail.com>,Matt
Arsenault <Matthew.Arsenault at amd.com>,David Green <david.green at arm.com>,Alan
Li <me at alanli.org>,Valentin Clement =?utf-8?b?KOODkOODrOODsw=?=,Alexey
Samsonov <vonosmas at gmail.com>,Jeong Jihyeon
<84846295+JihyeonJeong129 at users.noreply.github.com>,Anshul Nigham
<nigham at google.com>,Krzysztof Drewniak <Krzysztof.Drewniak at amd.com>,Thurston
Dang <thurston at google.com>,Bruno Cardoso Lopes <bruno.cardoso at gmail.com>,Florian
Hahn <flo at fhahn.com>,Craig Topper <craig.topper at sifive.com>,Yonah Goldberg
<ygoldberg at nvidia.com>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/224522 at github.com>
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/224522
>From 56182f872392cfd3e767eb3297e1cfe8df8a9272 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 21 Sep 2026 18:07:34 +0000
Subject: [PATCH 01/46] [GlobalISel] Add baseline mem.cache_hint tests
---
.../GlobalISel/irtranslator-mem-cache-hint.ll | 86 +++++++++++++++++++
.../GlobalISel/legalizer/mem-cache-hint.ll | 27 ++++++
2 files changed, 113 insertions(+)
create mode 100644 llvm/test/CodeGen/Generic/GlobalISel/irtranslator-mem-cache-hint.ll
create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/legalizer/mem-cache-hint.ll
diff --git a/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-mem-cache-hint.ll b/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-mem-cache-hint.ll
new file mode 100644
index 00000000000000..2e16281b4c8573
--- /dev/null
+++ b/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-mem-cache-hint.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=x86_64-linux-gnu -O0 -global-isel \
+; RUN: -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s
+
+; REQUIRES: x86-registered-target
+
+define i32 @load_store(ptr %p, i32 %v) {
+ ; CHECK-LABEL: name: load_store
+ ; CHECK: bb.1 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $esi, $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $rdi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi
+ ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32) from %ir.p)
+ ; CHECK-NEXT: G_STORE [[COPY1]](s32), [[COPY]](p0) :: (store (s32) into %ir.p)
+ ; CHECK-NEXT: $eax = COPY [[LOAD]](s32)
+ ; CHECK-NEXT: RET 0, implicit $eax
+ %load = load i32, ptr %p, align 4, !mem.cache_hint !0
+ store i32 %v, ptr %p, align 4, !mem.cache_hint !1
+ ret i32 %load
+}
+
+define i32 @atomics(ptr %p, i32 %v) {
+ ; CHECK-LABEL: name: atomics
+ ; CHECK: bb.1 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $esi, $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $rdi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi
+ ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load monotonic (s32) from %ir.p)
+ ; CHECK-NEXT: G_STORE [[COPY1]](s32), [[COPY]](p0) :: (store release (s32) into %ir.p)
+ ; CHECK-NEXT: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[COPY1]] :: (load store monotonic (s32) on %ir.p)
+ ; CHECK-NEXT: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p0), [[ATOMICRMW_ADD]], [[COPY1]] :: (load store seq_cst monotonic (s32) on %ir.p)
+ ; CHECK-NEXT: $eax = COPY [[ATOMIC_CMPXCHG_WITH_SUCCESS]](s32)
+ ; CHECK-NEXT: RET 0, implicit $eax
+ %load = load atomic i32, ptr %p monotonic, align 4, !mem.cache_hint !0
+ store atomic i32 %v, ptr %p release, align 4, !mem.cache_hint !1
+ %old = atomicrmw add ptr %p, i32 %v monotonic, align 4, !mem.cache_hint !0
+ %pair = cmpxchg ptr %p, i32 %old, i32 %v seq_cst monotonic, align 4, !mem.cache_hint !0
+ %result = extractvalue { i32, i1 } %pair, 0
+ ret i32 %result
+}
+
+define i32 @target_memory_intrinsic(ptr %p) {
+ ; CHECK-LABEL: name: target_memory_intrinsic
+ ; CHECK: bb.1 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $rdi
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.x86.atomic.bts), [[COPY]](p0), 1 :: (volatile load store (s32) on %ir.p, align 32)
+ ; CHECK-NEXT: $eax = COPY [[INT]](s32)
+ ; CHECK-NEXT: RET 0, implicit $eax
+ %old = call i32 @llvm.x86.atomic.bts.i32(ptr %p, i8 1), !mem.cache_hint !0
+ ret i32 %old
+}
+
+define void @memory_intrinsics(ptr %dst, ptr %src) {
+ ; CHECK-LABEL: name: memory_intrinsics
+ ; CHECK: bb.1 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi, $rsi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $rdi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $rsi
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s8) = G_CONSTANT i8 0
+ ; CHECK-NEXT: G_MEMCPY [[COPY]](p0), [[COPY1]](p0), [[C]](s64), 0 :: (store (s8) into %ir.dst, align 4), (load (s8) from %ir.src, align 4)
+ ; CHECK-NEXT: G_MEMMOVE [[COPY]](p0), [[COPY1]](p0), [[C]](s64), 0 :: (store (s8) into %ir.dst, align 4), (load (s8) from %ir.src, align 4)
+ ; CHECK-NEXT: G_MEMSET [[COPY]](p0), [[C1]](s8), [[C]](s64), 0 :: (store (s8) into %ir.dst, align 4)
+ ; CHECK-NEXT: RET 0
+ call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dst, ptr align 4 %src, i64 32, i1 false), !mem.cache_hint !2
+ call void @llvm.memmove.p0.p0.i64(ptr align 4 %dst, ptr align 4 %src, i64 32, i1 false), !mem.cache_hint !2
+ call void @llvm.memset.p0.i64(ptr align 4 %dst, i8 0, i64 32, i1 false), !mem.cache_hint !3
+ ret void
+}
+
+declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
+declare void @llvm.memmove.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1 immarg)
+declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)
+declare i32 @llvm.x86.atomic.bts.i32(ptr, i8 immarg)
+
+!0 = !{i32 0, !4}
+!1 = !{i32 1, !5}
+!2 = !{i32 0, !5, i32 1, !4}
+!3 = !{i32 0, !5}
+!4 = !{!"test.cache", !"load"}
+!5 = !{!"test.cache", !"store"}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mem-cache-hint.ll b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mem-cache-hint.ll
new file mode 100644
index 00000000000000..dc6f4f4c0d355a
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mem-cache-hint.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=mipsel-linux-gnu -O0 -global-isel -global-isel-abort=1 \
+; RUN: -stop-after=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+define void @split_load_store(ptr %src, ptr %dst) {
+ ; CHECK-LABEL: name: split_load_store
+ ; CHECK: bb.1 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $a0, $a1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
+ ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32) from %ir.src)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+ ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s32)
+ ; CHECK-NEXT: G_STORE [[LOAD]](s32), [[COPY1]](p0) :: (store (s16) into %ir.dst, align 4)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+ ; CHECK-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p0) :: (store (s8) into %ir.dst + 2, align 2, basealign 4)
+ ; CHECK-NEXT: RetRA
+ %value = load i24, ptr %src, align 4, !mem.cache_hint !0
+ store i24 %value, ptr %dst, align 4, !mem.cache_hint !1
+ ret void
+}
+
+!0 = !{i32 0, !2}
+!1 = !{i32 1, !2}
+!2 = !{!"test.cache", !"hint"}
>From 38f382bffa2efa1089dd10f13c7f26eedf176672 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 21 Sep 2026 18:09:13 +0000
Subject: [PATCH 02/46] [GlobalISel] Preserve mem.cache_hint metadata
---
.../CodeGen/GlobalISel/GenericMachineInstrs.h | 6 +--
llvm/include/llvm/CodeGen/TargetLowering.h | 9 ++++
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 54 ++++++++++++-------
llvm/lib/Target/X86/X86ISelLowering.cpp | 16 +++---
.../GlobalISel/irtranslator-mem-cache-hint.ll | 20 +++----
.../GlobalISel/legalizer/mem-cache-hint.ll | 6 +--
6 files changed, 69 insertions(+), 42 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 1a0c1840289ba6..3571f22efda74b 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -72,6 +72,9 @@ class GMemOperation : public GenericMachineInstr {
/// Returns the size in bits of the memory access.
LocationSize getMemSizeInBits() const { return getMMO().getSizeInBits(); }
+ /// Returns the cache hint metadata for this memory access.
+ const MDNode *getMemCacheHint() const { return getMMO().getMemCacheHint(); }
+
static bool classof(const MachineInstr *MI) {
return GenericMachineInstr::classof(MI) && MI->hasOneMemOperand();
}
@@ -192,9 +195,6 @@ class GAnyLoad : public GLoadStore {
return getMMO().getRanges();
}
- /// Returns the cache hint metadata for this load.
- const MDNode *getMemCacheHint() const { return getMMO().getMemCacheHint(); }
-
static bool classof(const MachineInstr *MI) {
switch (MI->getOpcode()) {
case TargetOpcode::G_LOAD:
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index c08d0e53ec34b3..2e123b5af96f04 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1263,6 +1263,15 @@ class LLVM_ABI TargetLoweringBase {
// value representing memory location
PointerUnion<const Value *, const PseudoSourceValue *> ptrVal;
+ // Index of the call argument represented by ptrVal. This allows
+ // per-argument metadata to be associated with this memory access.
+ std::optional<unsigned> ptrArgIndex;
+
+ void setPointerOperand(const CallBase &I, unsigned ArgNo) {
+ ptrVal = I.getArgOperand(ArgNo);
+ ptrArgIndex = ArgNo;
+ }
+
// Fallback address space for use if ptrVal is nullptr. std::nullopt means
// unknown address space.
std::optional<unsigned> fallbackAddressSpace;
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index d9e88f22ac2ca3..0cdd897b6cf7b0 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2139,6 +2139,7 @@ bool IRTranslatorImpl::translateLoad(const User &U,
ArrayRef<Register> Regs = getOrCreateVRegs(LI);
Register Base = getOrCreateVReg(*LI.getPointerOperand());
AAMDNodes AAInfo = LI.getAAMetadata();
+ const MDNode *MemCacheHint = getMemCacheHintMetadata(LI);
const Value *Ptr = LI.getPointerOperand();
@@ -2164,7 +2165,8 @@ bool IRTranslatorImpl::translateLoad(const User &U,
auto *MMO = MF->getMachineMemOperand(
MachinePointerInfo(LI.getPointerOperand()), Flags,
MRI->getType(Regs[0]), getMemOpAlign(LI),
- MMOMetadata(AAInfo, LI.getMetadata(LLVMContext::MD_range)),
+ MMOMetadata(AAInfo, LI.getMetadata(LLVMContext::MD_range),
+ MemCacheHint),
LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[0], Base, *MMO);
return true;
@@ -2181,7 +2183,8 @@ bool IRTranslatorImpl::translateLoad(const User &U,
Align BaseAlign = getMemOpAlign(LI);
auto *MMO =
MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Regs[i]),
- commonAlignment(BaseAlign, Offsets[i]), AAInfo,
+ commonAlignment(BaseAlign, Offsets[i]),
+ MMOMetadata(AAInfo, nullptr, MemCacheHint),
LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
}
@@ -2208,11 +2211,14 @@ bool IRTranslatorImpl::translateStore(const User &U,
}
MachineMemOperand::Flags Flags = TLI->getStoreMemOperandFlags(SI, *DL);
+ const MDNode *MemCacheHint =
+ getMemCacheHintMetadata(SI, SI.getPointerOperandIndex());
// Fast-path the common single-register store.
if (Vals.size() == 1) {
auto *MMO = MF->getMachineMemOperand(
MachinePointerInfo(SI.getPointerOperand()), Flags,
- MRI->getType(Vals[0]), getMemOpAlign(SI), SI.getAAMetadata(),
+ MRI->getType(Vals[0]), getMemOpAlign(SI),
+ MMOMetadata(SI.getAAMetadata(), nullptr, MemCacheHint),
SI.getSyncScopeID(), SI.getOrdering());
MIRBuilder.buildStore(Vals[0], Base, *MMO);
return true;
@@ -2227,10 +2233,11 @@ bool IRTranslatorImpl::translateStore(const User &U,
MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i]);
Align BaseAlign = getMemOpAlign(SI);
- auto *MMO = MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Vals[i]),
- commonAlignment(BaseAlign, Offsets[i]),
- SI.getAAMetadata(),
- SI.getSyncScopeID(), SI.getOrdering());
+ auto *MMO = MF->getMachineMemOperand(
+ Ptr, Flags, MRI->getType(Vals[i]),
+ commonAlignment(BaseAlign, Offsets[i]),
+ MMOMetadata(SI.getAAMetadata(), nullptr, MemCacheHint),
+ SI.getSyncScopeID(), SI.getOrdering());
MIRBuilder.buildStore(Vals[i], Addr, *MMO);
}
return true;
@@ -2573,6 +2580,8 @@ bool IRTranslatorImpl::translateMemFunc(const CallInst &CI,
}
AAMDNodes AAInfo = CI.getAAMetadata();
+ const MDNode *DstMemCacheHint = getMemCacheHintMetadata(CI, /*OperandNo=*/0);
+ const MDNode *SrcMemCacheHint = getMemCacheHintMetadata(CI, /*OperandNo=*/1);
if (AA && CopySize &&
AA->pointsToConstantMemory(MemoryLocation(
SrcPtr, LocationSize::precise(CopySize->getZExtValue()), AAInfo))) {
@@ -2584,13 +2593,14 @@ bool IRTranslatorImpl::translateMemFunc(const CallInst &CI,
LoadFlags |= MachineMemOperand::MODereferenceable;
}
- ICall.addMemOperand(
- MF->getMachineMemOperand(MachinePointerInfo(CI.getArgOperand(0)),
- StoreFlags, 1, DstAlign, AAInfo));
+ ICall.addMemOperand(MF->getMachineMemOperand(
+ MachinePointerInfo(CI.getArgOperand(0)), StoreFlags, 1, DstAlign,
+ MMOMetadata(AAInfo, nullptr, DstMemCacheHint)));
if (Opcode != TargetOpcode::G_MEMSET &&
Opcode != TargetOpcode::G_MEMSET_INLINE)
ICall.addMemOperand(MF->getMachineMemOperand(
- MachinePointerInfo(SrcPtr), LoadFlags, 1, SrcAlign, AAInfo));
+ MachinePointerInfo(SrcPtr), LoadFlags, 1, SrcAlign,
+ MMOMetadata(AAInfo, nullptr, SrcMemCacheHint)));
return true;
}
@@ -3744,8 +3754,12 @@ bool IRTranslatorImpl::translateIntrinsic(
} else if (Info.fallbackAddressSpace) {
MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
}
+ const MDNode *MemCacheHint =
+ Info.ptrArgIndex ? getMemCacheHintMetadata(CB, *Info.ptrArgIndex)
+ : nullptr;
MIB.addMemOperand(MF->getMachineMemOperand(
- MPI, Info.flags, MemTy, Alignment, CB.getAAMetadata(), Info.ssid,
+ MPI, Info.flags, MemTy, Alignment,
+ MMOMetadata(CB.getAAMetadata(), nullptr, MemCacheHint), Info.ssid,
Info.order, Info.failureOrder));
}
@@ -4371,8 +4385,10 @@ bool IRTranslatorImpl::translateAtomicCmpXchg(const User &U,
OldValRes, SuccessRes, Addr, Cmp, NewVal,
*MF->getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
- getMemOpAlign(I), I.getAAMetadata(), I.getSyncScopeID(),
- I.getSuccessOrdering(), I.getFailureOrdering()));
+ getMemOpAlign(I),
+ MMOMetadata(I.getAAMetadata(), nullptr,
+ getMemCacheHintMetadata(I, I.getPointerOperandIndex())),
+ I.getSyncScopeID(), I.getSuccessOrdering(), I.getFailureOrdering()));
return true;
}
@@ -4465,10 +4481,12 @@ bool IRTranslatorImpl::translateAtomicRMW(const User &U,
MIRBuilder.buildAtomicRMW(
Opcode, Res, Addr, Val,
- *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
- Flags, MRI->getType(Val), getMemOpAlign(I),
- I.getAAMetadata(), I.getSyncScopeID(),
- I.getOrdering()));
+ *MF->getMachineMemOperand(
+ MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Val),
+ getMemOpAlign(I),
+ MMOMetadata(I.getAAMetadata(), nullptr,
+ getMemCacheHintMetadata(I, I.getPointerOperandIndex())),
+ I.getSyncScopeID(), I.getOrdering()));
return true;
}
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7b3af6152fcfe1..818d464f75de52 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -3383,7 +3383,7 @@ void X86TargetLowering::getTgtMemIntrinsic(
case Intrinsic::x86_aesenc128kl:
case Intrinsic::x86_aesdec128kl:
Info.opc = ISD::INTRINSIC_W_CHAIN;
- Info.ptrVal = I.getArgOperand(1);
+ Info.setPointerOperand(I, 1);
Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
Info.align = Align(1);
Info.flags |= MachineMemOperand::MOLoad;
@@ -3392,7 +3392,7 @@ void X86TargetLowering::getTgtMemIntrinsic(
case Intrinsic::x86_aesenc256kl:
case Intrinsic::x86_aesdec256kl:
Info.opc = ISD::INTRINSIC_W_CHAIN;
- Info.ptrVal = I.getArgOperand(1);
+ Info.setPointerOperand(I, 1);
Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
Info.align = Align(1);
Info.flags |= MachineMemOperand::MOLoad;
@@ -3401,7 +3401,7 @@ void X86TargetLowering::getTgtMemIntrinsic(
case Intrinsic::x86_aesencwide128kl:
case Intrinsic::x86_aesdecwide128kl:
Info.opc = ISD::INTRINSIC_W_CHAIN;
- Info.ptrVal = I.getArgOperand(0);
+ Info.setPointerOperand(I, 0);
Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
Info.align = Align(1);
Info.flags |= MachineMemOperand::MOLoad;
@@ -3410,7 +3410,7 @@ void X86TargetLowering::getTgtMemIntrinsic(
case Intrinsic::x86_aesencwide256kl:
case Intrinsic::x86_aesdecwide256kl:
Info.opc = ISD::INTRINSIC_W_CHAIN;
- Info.ptrVal = I.getArgOperand(0);
+ Info.setPointerOperand(I, 0);
Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
Info.align = Align(1);
Info.flags |= MachineMemOperand::MOLoad;
@@ -3422,7 +3422,7 @@ void X86TargetLowering::getTgtMemIntrinsic(
case Intrinsic::x86_atomic_btc:
case Intrinsic::x86_atomic_btr: {
Info.opc = ISD::INTRINSIC_W_CHAIN;
- Info.ptrVal = I.getArgOperand(0);
+ Info.setPointerOperand(I, 0);
unsigned Size = I.getType()->getScalarSizeInBits();
Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
Info.align = Align(Size);
@@ -3435,7 +3435,7 @@ void X86TargetLowering::getTgtMemIntrinsic(
case Intrinsic::x86_atomic_btc_rm:
case Intrinsic::x86_atomic_btr_rm: {
Info.opc = ISD::INTRINSIC_W_CHAIN;
- Info.ptrVal = I.getArgOperand(0);
+ Info.setPointerOperand(I, 0);
unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
Info.align = Align(Size);
@@ -3458,7 +3458,7 @@ void X86TargetLowering::getTgtMemIntrinsic(
case Intrinsic::x86_atomic_and_cc:
case Intrinsic::x86_atomic_xor_cc: {
Info.opc = ISD::INTRINSIC_W_CHAIN;
- Info.ptrVal = I.getArgOperand(0);
+ Info.setPointerOperand(I, 0);
unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
Info.align = Align(Size);
@@ -3476,7 +3476,7 @@ void X86TargetLowering::getTgtMemIntrinsic(
case TRUNCATE_TO_MEM_VI16:
case TRUNCATE_TO_MEM_VI32: {
Info.opc = ISD::INTRINSIC_VOID;
- Info.ptrVal = I.getArgOperand(0);
+ Info.setPointerOperand(I, 0);
MVT VT = MVT::getVT(I.getArgOperand(1)->getType());
MVT ScalarVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
if (IntrData->Type == TRUNCATE_TO_MEM_VI8)
diff --git a/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-mem-cache-hint.ll b/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-mem-cache-hint.ll
index 2e16281b4c8573..160b2842937b46 100644
--- a/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-mem-cache-hint.ll
+++ b/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-mem-cache-hint.ll
@@ -11,8 +11,8 @@ define i32 @load_store(ptr %p, i32 %v) {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $rdi
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi
- ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32) from %ir.p)
- ; CHECK-NEXT: G_STORE [[COPY1]](s32), [[COPY]](p0) :: (store (s32) into %ir.p)
+ ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32) from %ir.p, !mem.cache_hint !1)
+ ; CHECK-NEXT: G_STORE [[COPY1]](s32), [[COPY]](p0) :: (store (s32) into %ir.p, !mem.cache_hint !3)
; CHECK-NEXT: $eax = COPY [[LOAD]](s32)
; CHECK-NEXT: RET 0, implicit $eax
%load = load i32, ptr %p, align 4, !mem.cache_hint !0
@@ -27,10 +27,10 @@ define i32 @atomics(ptr %p, i32 %v) {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $rdi
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi
- ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load monotonic (s32) from %ir.p)
- ; CHECK-NEXT: G_STORE [[COPY1]](s32), [[COPY]](p0) :: (store release (s32) into %ir.p)
- ; CHECK-NEXT: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[COPY1]] :: (load store monotonic (s32) on %ir.p)
- ; CHECK-NEXT: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p0), [[ATOMICRMW_ADD]], [[COPY1]] :: (load store seq_cst monotonic (s32) on %ir.p)
+ ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load monotonic (s32) from %ir.p, !mem.cache_hint !1)
+ ; CHECK-NEXT: G_STORE [[COPY1]](s32), [[COPY]](p0) :: (store release (s32) into %ir.p, !mem.cache_hint !3)
+ ; CHECK-NEXT: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[COPY1]] :: (load store monotonic (s32) on %ir.p, !mem.cache_hint !1)
+ ; CHECK-NEXT: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p0), [[ATOMICRMW_ADD]], [[COPY1]] :: (load store seq_cst monotonic (s32) on %ir.p, !mem.cache_hint !1)
; CHECK-NEXT: $eax = COPY [[ATOMIC_CMPXCHG_WITH_SUCCESS]](s32)
; CHECK-NEXT: RET 0, implicit $eax
%load = load atomic i32, ptr %p monotonic, align 4, !mem.cache_hint !0
@@ -47,7 +47,7 @@ define i32 @target_memory_intrinsic(ptr %p) {
; CHECK-NEXT: liveins: $rdi
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $rdi
- ; CHECK-NEXT: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.x86.atomic.bts), [[COPY]](p0), 1 :: (volatile load store (s32) on %ir.p, align 32)
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.x86.atomic.bts), [[COPY]](p0), 1 :: (volatile load store (s32) on %ir.p, align 32, !mem.cache_hint !1)
; CHECK-NEXT: $eax = COPY [[INT]](s32)
; CHECK-NEXT: RET 0, implicit $eax
%old = call i32 @llvm.x86.atomic.bts.i32(ptr %p, i8 1), !mem.cache_hint !0
@@ -63,9 +63,9 @@ define void @memory_intrinsics(ptr %dst, ptr %src) {
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $rsi
; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s8) = G_CONSTANT i8 0
- ; CHECK-NEXT: G_MEMCPY [[COPY]](p0), [[COPY1]](p0), [[C]](s64), 0 :: (store (s8) into %ir.dst, align 4), (load (s8) from %ir.src, align 4)
- ; CHECK-NEXT: G_MEMMOVE [[COPY]](p0), [[COPY1]](p0), [[C]](s64), 0 :: (store (s8) into %ir.dst, align 4), (load (s8) from %ir.src, align 4)
- ; CHECK-NEXT: G_MEMSET [[COPY]](p0), [[C1]](s8), [[C]](s64), 0 :: (store (s8) into %ir.dst, align 4)
+ ; CHECK-NEXT: G_MEMCPY [[COPY]](p0), [[COPY1]](p0), [[C]](s64), 0 :: (store (s8) into %ir.dst, align 4, !mem.cache_hint !3), (load (s8) from %ir.src, align 4, !mem.cache_hint !1)
+ ; CHECK-NEXT: G_MEMMOVE [[COPY]](p0), [[COPY1]](p0), [[C]](s64), 0 :: (store (s8) into %ir.dst, align 4, !mem.cache_hint !3), (load (s8) from %ir.src, align 4, !mem.cache_hint !1)
+ ; CHECK-NEXT: G_MEMSET [[COPY]](p0), [[C1]](s8), [[C]](s64), 0 :: (store (s8) into %ir.dst, align 4, !mem.cache_hint !3)
; CHECK-NEXT: RET 0
call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dst, ptr align 4 %src, i64 32, i1 false), !mem.cache_hint !2
call void @llvm.memmove.p0.p0.i64(ptr align 4 %dst, ptr align 4 %src, i64 32, i1 false), !mem.cache_hint !2
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mem-cache-hint.ll b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mem-cache-hint.ll
index dc6f4f4c0d355a..5550b66ab9e6f1 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mem-cache-hint.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mem-cache-hint.ll
@@ -9,13 +9,13 @@ define void @split_load_store(ptr %src, ptr %dst) {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
- ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32) from %ir.src)
+ ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32) from %ir.src, !mem.cache_hint !1)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: G_STORE [[LOAD]](s32), [[COPY1]](p0) :: (store (s16) into %ir.dst, align 4)
+ ; CHECK-NEXT: G_STORE [[LOAD]](s32), [[COPY1]](p0) :: (store (s16) into %ir.dst, align 4, !mem.cache_hint !1)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
- ; CHECK-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p0) :: (store (s8) into %ir.dst + 2, align 2, basealign 4)
+ ; CHECK-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p0) :: (store (s8) into %ir.dst + 2, align 2, basealign 4, !mem.cache_hint !1)
; CHECK-NEXT: RetRA
%value = load i24, ptr %src, align 4, !mem.cache_hint !0
store i24 %value, ptr %dst, align 4, !mem.cache_hint !1
>From 4d02eb1b6091956ee664683b46331d2c7714321e Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 21 Sep 2026 19:32:54 +0000
Subject: [PATCH 03/46] [GlobalISel] Move cache hint IRTranslator test under
X86
---
.../GlobalISel/irtranslator-mem-cache-hint.ll | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
rename llvm/test/CodeGen/{Generic => X86}/GlobalISel/irtranslator-mem-cache-hint.ll (97%)
diff --git a/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-mem-cache-hint.ll b/llvm/test/CodeGen/X86/GlobalISel/irtranslator-mem-cache-hint.ll
similarity index 97%
rename from llvm/test/CodeGen/Generic/GlobalISel/irtranslator-mem-cache-hint.ll
rename to llvm/test/CodeGen/X86/GlobalISel/irtranslator-mem-cache-hint.ll
index 160b2842937b46..4aeb4cf6ed5a1d 100644
--- a/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-mem-cache-hint.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/irtranslator-mem-cache-hint.ll
@@ -1,8 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=x86_64-linux-gnu -O0 -global-isel \
-; RUN: -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s
-
-; REQUIRES: x86-registered-target
+; RUN: -stop-after=irtranslator %s -o - | FileCheck %s
define i32 @load_store(ptr %p, i32 %v) {
; CHECK-LABEL: name: load_store
>From ef5eed9597dc29cc6e7d22361a6a6aca17c09274 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 30 Sep 2026 19:43:03 +0000
Subject: [PATCH 04/46] [GlobalISel] Check cache hint metadata contents in MIR
tests
---
llvm/test/CodeGen/Mips/GlobalISel/legalizer/mem-cache-hint.ll | 2 ++
.../test/CodeGen/X86/GlobalISel/irtranslator-mem-cache-hint.ll | 3 +++
2 files changed, 5 insertions(+)
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mem-cache-hint.ll b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mem-cache-hint.ll
index 5550b66ab9e6f1..0f886e8b05027c 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mem-cache-hint.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/mem-cache-hint.ll
@@ -2,6 +2,8 @@
; RUN: llc -mtriple=mipsel-linux-gnu -O0 -global-isel -global-isel-abort=1 \
; RUN: -stop-after=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+; CHECK-DAG: !1 = !{!"test.cache", !"hint"}
+
define void @split_load_store(ptr %src, ptr %dst) {
; CHECK-LABEL: name: split_load_store
; CHECK: bb.1 (%ir-block.0):
diff --git a/llvm/test/CodeGen/X86/GlobalISel/irtranslator-mem-cache-hint.ll b/llvm/test/CodeGen/X86/GlobalISel/irtranslator-mem-cache-hint.ll
index 4aeb4cf6ed5a1d..b0e2a3289dee00 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/irtranslator-mem-cache-hint.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/irtranslator-mem-cache-hint.ll
@@ -2,6 +2,9 @@
; RUN: llc -mtriple=x86_64-linux-gnu -O0 -global-isel \
; RUN: -stop-after=irtranslator %s -o - | FileCheck %s
+; CHECK-DAG: !1 = !{!"test.cache", !"load"}
+; CHECK-DAG: !3 = !{!"test.cache", !"store"}
+
define i32 @load_store(ptr %p, i32 %v) {
; CHECK-LABEL: name: load_store
; CHECK: bb.1 (%ir-block.0):
>From 2f828dece5e875466ef67d41b0a5c4fd6d555f89 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Tue, 6 Oct 2026 15:31:41 -0400
Subject: [PATCH 05/46] [libc++] Pass -fsized-deallocation to the sized_delete
tests on AppleClang 23 (#229257)
AppleClang 23 still defaults to -fno-sized-deallocation, so explicitly
pass -fsized-deallocation in the tests that require it.
---
.../new.delete/new.delete.array/sized_delete_array.pass.cpp | 2 ++
.../new.delete/new.delete.single/sized_delete.pass.cpp | 2 ++
2 files changed, 4 insertions(+)
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array.pass.cpp
index 0e411f8adc0f2a..d72ef35b9d494f 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array.pass.cpp
@@ -12,6 +12,8 @@
// These compiler versions and platforms don't enable sized deallocation by default.
// ADDITIONAL_COMPILE_FLAGS(apple-clang-21): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-22): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-23): -fsized-deallocation
// ADDITIONAL_COMPILE_FLAGS(target=x86_64-w64-windows-gnu): -fsized-deallocation
// ADDITIONAL_COMPILE_FLAGS(target=i686-w64-windows-gnu): -fsized-deallocation
// ADDITIONAL_COMPILE_FLAGS(target=aarch64-w64-windows-gnu): -fsized-deallocation
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete.pass.cpp
index ca0f612bf75788..63615d65ec2034 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete.pass.cpp
@@ -12,6 +12,8 @@
// These compiler versions and platforms don't enable sized deallocation by default.
// ADDITIONAL_COMPILE_FLAGS(apple-clang-21): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-22): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-23): -fsized-deallocation
// ADDITIONAL_COMPILE_FLAGS(target=x86_64-w64-windows-gnu): -fsized-deallocation
// ADDITIONAL_COMPILE_FLAGS(target=i686-w64-windows-gnu): -fsized-deallocation
// ADDITIONAL_COMPILE_FLAGS(target=aarch64-w64-windows-gnu): -fsized-deallocation
>From 858aa35450ef48a0a7444bedf865936e824fb1ee Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Tue, 6 Oct 2026 21:40:35 +0200
Subject: [PATCH 06/46] AMDGPU: Fix SILoadStoreOptimizer dropping gds bit on DS
merges (#229520)
When forming read2/write2 from a pair of DS_READ/DS_WRITE
instructions, the gds operand was unconditionally set to 0, turning
GDS accesses into LDS accesses. Preserve the gds bit, refuse to pair
an LDS access with a GDS access, and select the M0-reading opcode
variant for GDS on targets that otherwise use the _gfx9 forms.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
.../Target/AMDGPU/SILoadStoreOptimizer.cpp | 42 +++--
.../AMDGPU/load-store-opt-ds-gds-gfx7.mir | 170 +++++++++++++++++
.../AMDGPU/load-store-opt-ds-gds-gfx9.mir | 171 ++++++++++++++++++
3 files changed, 366 insertions(+), 17 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/load-store-opt-ds-gds-gfx7.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/load-store-opt-ds-gds-gfx9.mir
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 890b80f6639b8d..8ac7674dd3e3f8 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -118,6 +118,7 @@ class SILoadStoreOptimizer {
unsigned DMask;
InstClassEnum InstClass;
unsigned CPol = 0;
+ bool GDS = false;
const TargetRegisterClass *DataRC;
bool UseST64;
int AddrIdx[MaxAddressRegs];
@@ -240,14 +241,14 @@ class SILoadStoreOptimizer {
MachineBasicBlock::iterator InsertBefore,
const DebugLoc &DL, AMDGPU::OpName OpName) const;
- unsigned read2Opcode(unsigned EltSize) const;
- unsigned read2ST64Opcode(unsigned EltSize) const;
+ unsigned read2Opcode(unsigned EltSize, bool GDS) const;
+ unsigned read2ST64Opcode(unsigned EltSize, bool GDS) const;
MachineBasicBlock::iterator
mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
MachineBasicBlock::iterator InsertBefore);
- unsigned write2Opcode(unsigned EltSize) const;
- unsigned write2ST64Opcode(unsigned EltSize) const;
+ unsigned write2Opcode(unsigned EltSize, bool GDS) const;
+ unsigned write2ST64Opcode(unsigned EltSize, bool GDS) const;
unsigned getWrite2Opcode(const CombineInfo &CI) const;
MachineBasicBlock::iterator
@@ -897,6 +898,7 @@ void SILoadStoreOptimizer::CombineInfo::setMI(MachineBasicBlock::iterator MI,
if ((InstClass == DS_READ) || (InstClass == DS_WRITE)) {
Offset &= 0xffff;
+ GDS = LSO.TII->getNamedOperand(*I, AMDGPU::OpName::gds)->getImm();
} else if (InstClass != MIMG) {
CPol = LSO.TII->getNamedOperand(*I, AMDGPU::OpName::cpol)->getImm();
}
@@ -1092,6 +1094,9 @@ bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI,
if (CI.Offset == Paired.Offset)
return false;
+ if (CI.GDS != Paired.GDS)
+ return false;
+
// This won't be valid if the offset isn't aligned.
if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0))
return false;
@@ -1428,14 +1433,15 @@ SILoadStoreOptimizer::copyFromSrcRegs(CombineInfo &CI, CombineInfo &Paired,
return SrcReg;
}
-unsigned SILoadStoreOptimizer::read2Opcode(unsigned EltSize) const {
- if (STM->ldsRequiresM0Init())
+unsigned SILoadStoreOptimizer::read2Opcode(unsigned EltSize, bool GDS) const {
+ if (GDS || STM->ldsRequiresM0Init())
return (EltSize == 4) ? AMDGPU::DS_READ2_B32 : AMDGPU::DS_READ2_B64;
return (EltSize == 4) ? AMDGPU::DS_READ2_B32_gfx9 : AMDGPU::DS_READ2_B64_gfx9;
}
-unsigned SILoadStoreOptimizer::read2ST64Opcode(unsigned EltSize) const {
- if (STM->ldsRequiresM0Init())
+unsigned SILoadStoreOptimizer::read2ST64Opcode(unsigned EltSize,
+ bool GDS) const {
+ if (GDS || STM->ldsRequiresM0Init())
return (EltSize == 4) ? AMDGPU::DS_READ2ST64_B32 : AMDGPU::DS_READ2ST64_B64;
return (EltSize == 4) ? AMDGPU::DS_READ2ST64_B32_gfx9
@@ -1453,8 +1459,8 @@ SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
unsigned NewOffset0 = std::min(CI.Offset, Paired.Offset);
unsigned NewOffset1 = std::max(CI.Offset, Paired.Offset);
- unsigned Opc =
- CI.UseST64 ? read2ST64Opcode(CI.EltSize) : read2Opcode(CI.EltSize);
+ unsigned Opc = CI.UseST64 ? read2ST64Opcode(CI.EltSize, CI.GDS)
+ : read2Opcode(CI.EltSize, CI.GDS);
assert((isUInt<8>(NewOffset0) && isUInt<8>(NewOffset1)) &&
(NewOffset0 != NewOffset1) && "Computed offset doesn't fit");
@@ -1490,7 +1496,7 @@ SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
.addReg(BaseReg, BaseRegFlags, BaseSubReg) // addr
.addImm(NewOffset0) // offset0
.addImm(NewOffset1) // offset1
- .addImm(0) // gds
+ .addImm(CI.GDS) // gds
.cloneMergedMemRefs({&*CI.I, &*Paired.I});
copyToDestRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdst, DestReg);
@@ -1502,15 +1508,16 @@ SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
return Read2;
}
-unsigned SILoadStoreOptimizer::write2Opcode(unsigned EltSize) const {
- if (STM->ldsRequiresM0Init())
+unsigned SILoadStoreOptimizer::write2Opcode(unsigned EltSize, bool GDS) const {
+ if (GDS || STM->ldsRequiresM0Init())
return (EltSize == 4) ? AMDGPU::DS_WRITE2_B32 : AMDGPU::DS_WRITE2_B64;
return (EltSize == 4) ? AMDGPU::DS_WRITE2_B32_gfx9
: AMDGPU::DS_WRITE2_B64_gfx9;
}
-unsigned SILoadStoreOptimizer::write2ST64Opcode(unsigned EltSize) const {
- if (STM->ldsRequiresM0Init())
+unsigned SILoadStoreOptimizer::write2ST64Opcode(unsigned EltSize,
+ bool GDS) const {
+ if (GDS || STM->ldsRequiresM0Init())
return (EltSize == 4) ? AMDGPU::DS_WRITE2ST64_B32
: AMDGPU::DS_WRITE2ST64_B64;
@@ -1519,7 +1526,8 @@ unsigned SILoadStoreOptimizer::write2ST64Opcode(unsigned EltSize) const {
}
unsigned SILoadStoreOptimizer::getWrite2Opcode(const CombineInfo &CI) const {
- return CI.UseST64 ? write2ST64Opcode(CI.EltSize) : write2Opcode(CI.EltSize);
+ return CI.UseST64 ? write2ST64Opcode(CI.EltSize, CI.GDS)
+ : write2Opcode(CI.EltSize, CI.GDS);
}
MachineBasicBlock::iterator SILoadStoreOptimizer::mergeWrite2Pair(
@@ -1578,7 +1586,7 @@ MachineBasicBlock::iterator SILoadStoreOptimizer::mergeWrite2Pair(
.add(*Data1) // data1
.addImm(NewOffset0) // offset0
.addImm(NewOffset1) // offset1
- .addImm(0) // gds
+ .addImm(CI.GDS) // gds
.cloneMergedMemRefs({&*CI.I, &*Paired.I});
CI.I->eraseFromParent();
diff --git a/llvm/test/CodeGen/AMDGPU/load-store-opt-ds-gds-gfx7.mir b/llvm/test/CodeGen/AMDGPU/load-store-opt-ds-gds-gfx7.mir
new file mode 100644
index 00000000000000..1de8c44531a213
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/load-store-opt-ds-gds-gfx7.mir
@@ -0,0 +1,170 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu7.04 -run-pass=si-load-store-opt -o - %s | FileCheck %s
+
+---
+name: ds_read_b32_gds_x2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ ; CHECK-LABEL: name: ds_read_b32_gds_x2
+ ; CHECK: liveins: $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: [[DS_READ2_B32_:%[0-9]+]]:vreg_64 = DS_READ2_B32 [[COPY]], 0, 1, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[DS_READ2_B32_]].sub0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY killed [[DS_READ2_B32_]].sub1
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]], implicit [[COPY2]]
+ %0:vgpr_32 = COPY $vgpr0
+ $m0 = S_MOV_B32 0
+ %1:vgpr_32 = DS_READ_B32 %0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ %2:vgpr_32 = DS_READ_B32 %0, 4, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ S_ENDPGM 0, implicit %1, implicit %2
+
+...
+---
+name: ds_read2st64_b32_gds_x2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ ; CHECK-LABEL: name: ds_read2st64_b32_gds_x2
+ ; CHECK: liveins: $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: [[DS_READ2ST64_B32_:%[0-9]+]]:vreg_64 = DS_READ2ST64_B32 [[COPY]], 0, 4, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[DS_READ2ST64_B32_]].sub0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY killed [[DS_READ2ST64_B32_]].sub1
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]], implicit [[COPY2]]
+ %0:vgpr_32 = COPY $vgpr0
+ $m0 = S_MOV_B32 0
+ %1:vgpr_32 = DS_READ_B32 %0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ %2:vgpr_32 = DS_READ_B32 %0, 1024, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ S_ENDPGM 0, implicit %1, implicit %2
+
+...
+---
+name: ds_read_b64_gds_x2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ ; CHECK-LABEL: name: ds_read_b64_gds_x2
+ ; CHECK: liveins: $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 0, 1, 1, implicit $m0, implicit $exec :: (load (s64), addrspace 2)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY [[DS_READ2_B64_]].sub0_sub1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY killed [[DS_READ2_B64_]].sub2_sub3
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]], implicit [[COPY2]]
+ %0:vgpr_32 = COPY $vgpr0
+ $m0 = S_MOV_B32 0
+ %1:vreg_64 = DS_READ_B64 %0, 0, 1, implicit $m0, implicit $exec :: (load (s64), addrspace 2)
+ %2:vreg_64 = DS_READ_B64 %0, 8, 1, implicit $m0, implicit $exec :: (load (s64), addrspace 2)
+ S_ENDPGM 0, implicit %1, implicit %2
+
+...
+---
+name: ds_write_b32_gds_x2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2
+
+ ; CHECK-LABEL: name: ds_write_b32_gds_x2
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: DS_WRITE2_B32 [[COPY]], [[COPY1]], [[COPY2]], 0, 1, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = COPY $vgpr2
+ $m0 = S_MOV_B32 0
+ DS_WRITE_B32 %0, %1, 0, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ DS_WRITE_B32 %0, %2, 4, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ S_ENDPGM 0
+
+...
+---
+name: ds_write2st64_b32_gds_x2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2
+
+ ; CHECK-LABEL: name: ds_write2st64_b32_gds_x2
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: DS_WRITE2ST64_B32 [[COPY]], [[COPY1]], [[COPY2]], 0, 4, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = COPY $vgpr2
+ $m0 = S_MOV_B32 0
+ DS_WRITE_B32 %0, %1, 0, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ DS_WRITE_B32 %0, %2, 1024, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ S_ENDPGM 0
+
+...
+---
+name: ds_read_b32_lds_gds
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ ; CHECK-LABEL: name: ds_read_b32_lds_gds
+ ; CHECK: liveins: $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: $m0 = S_MOV_B32 -1
+ ; CHECK-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (s32), addrspace 3)
+ ; CHECK-NEXT: [[DS_READ_B32_1:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 4, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[DS_READ_B32_]], implicit [[DS_READ_B32_1]]
+ %0:vgpr_32 = COPY $vgpr0
+ $m0 = S_MOV_B32 -1
+ %1:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec :: (load (s32), addrspace 3)
+ %2:vgpr_32 = DS_READ_B32 %0, 4, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ S_ENDPGM 0, implicit %1, implicit %2
+
+...
+---
+name: ds_write_b32_lds_gds
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2
+
+ ; CHECK-LABEL: name: ds_write_b32_lds_gds
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; CHECK-NEXT: $m0 = S_MOV_B32 -1
+ ; CHECK-NEXT: DS_WRITE_B32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (store (s32), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B32 [[COPY]], [[COPY2]], 4, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = COPY $vgpr2
+ $m0 = S_MOV_B32 -1
+ DS_WRITE_B32 %0, %1, 0, 0, implicit $m0, implicit $exec :: (store (s32), addrspace 3)
+ DS_WRITE_B32 %0, %2, 4, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/load-store-opt-ds-gds-gfx9.mir b/llvm/test/CodeGen/AMDGPU/load-store-opt-ds-gds-gfx9.mir
new file mode 100644
index 00000000000000..6991049719a892
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/load-store-opt-ds-gds-gfx9.mir
@@ -0,0 +1,171 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.00 -run-pass=si-load-store-opt -o - %s | FileCheck %s
+
+---
+name: ds_read_b32_gds_x2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ ; CHECK-LABEL: name: ds_read_b32_gds_x2
+ ; CHECK: liveins: $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: [[DS_READ2_B32_:%[0-9]+]]:vreg_64 = DS_READ2_B32 [[COPY]], 0, 1, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[DS_READ2_B32_]].sub0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY killed [[DS_READ2_B32_]].sub1
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]], implicit [[COPY2]]
+ %0:vgpr_32 = COPY $vgpr0
+ $m0 = S_MOV_B32 0
+ %1:vgpr_32 = DS_READ_B32 %0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ %2:vgpr_32 = DS_READ_B32 %0, 4, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ S_ENDPGM 0, implicit %1, implicit %2
+
+...
+---
+name: ds_read2st64_b32_gds_x2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ ; CHECK-LABEL: name: ds_read2st64_b32_gds_x2
+ ; CHECK: liveins: $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: [[DS_READ2ST64_B32_:%[0-9]+]]:vreg_64 = DS_READ2ST64_B32 [[COPY]], 0, 4, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[DS_READ2ST64_B32_]].sub0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY killed [[DS_READ2ST64_B32_]].sub1
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]], implicit [[COPY2]]
+ %0:vgpr_32 = COPY $vgpr0
+ $m0 = S_MOV_B32 0
+ %1:vgpr_32 = DS_READ_B32 %0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ %2:vgpr_32 = DS_READ_B32 %0, 1024, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ S_ENDPGM 0, implicit %1, implicit %2
+
+...
+---
+name: ds_read_b64_gds_x2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ ; CHECK-LABEL: name: ds_read_b64_gds_x2
+ ; CHECK: liveins: $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 0, 1, 1, implicit $m0, implicit $exec :: (load (s64), addrspace 2)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY [[DS_READ2_B64_]].sub0_sub1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY killed [[DS_READ2_B64_]].sub2_sub3
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]], implicit [[COPY2]]
+ %0:vgpr_32 = COPY $vgpr0
+ $m0 = S_MOV_B32 0
+ %1:vreg_64 = DS_READ_B64 %0, 0, 1, implicit $m0, implicit $exec :: (load (s64), addrspace 2)
+ %2:vreg_64 = DS_READ_B64 %0, 8, 1, implicit $m0, implicit $exec :: (load (s64), addrspace 2)
+ S_ENDPGM 0, implicit %1, implicit %2
+
+...
+---
+name: ds_write_b32_gds_x2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2
+
+ ; CHECK-LABEL: name: ds_write_b32_gds_x2
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: DS_WRITE2_B32 [[COPY]], [[COPY1]], [[COPY2]], 0, 1, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = COPY $vgpr2
+ $m0 = S_MOV_B32 0
+ DS_WRITE_B32 %0, %1, 0, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ DS_WRITE_B32 %0, %2, 4, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ S_ENDPGM 0
+
+...
+---
+name: ds_write2st64_b32_gds_x2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2
+
+ ; CHECK-LABEL: name: ds_write2st64_b32_gds_x2
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: DS_WRITE2ST64_B32 [[COPY]], [[COPY1]], [[COPY2]], 0, 4, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = COPY $vgpr2
+ $m0 = S_MOV_B32 0
+ DS_WRITE_B32 %0, %1, 0, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ DS_WRITE_B32 %0, %2, 1024, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ S_ENDPGM 0
+
+...
+---
+name: ds_read_b32_lds_gds
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ ; CHECK-LABEL: name: ds_read_b32_lds_gds
+ ; CHECK: liveins: $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_B32_gfx9 [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ ; CHECK-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 4, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[DS_READ_B32_gfx9_]], implicit [[DS_READ_B32_]]
+ %0:vgpr_32 = COPY $vgpr0
+ $m0 = S_MOV_B32 0
+ %1:vgpr_32 = DS_READ_B32_gfx9 %0, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ %2:vgpr_32 = DS_READ_B32 %0, 4, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 2)
+ S_ENDPGM 0, implicit %1, implicit %2
+
+...
+
+---
+name: ds_write_b32_lds_gds
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2
+
+ ; CHECK-LABEL: name: ds_write_b32_lds_gds
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; CHECK-NEXT: $m0 = S_MOV_B32 -1
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (store (s32), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B32 [[COPY]], [[COPY2]], 4, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = COPY $vgpr2
+ $m0 = S_MOV_B32 -1
+ DS_WRITE_B32_gfx9 %0, %1, 0, 0, implicit $exec :: (store (s32), addrspace 3)
+ DS_WRITE_B32 %0, %2, 4, 1, implicit $m0, implicit $exec :: (store (s32), addrspace 2)
+ S_ENDPGM 0
+
+...
>From b4884e468a45ed402f25dd2abb0d0cc89c4a15d8 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Tue, 6 Oct 2026 21:50:26 +0200
Subject: [PATCH 07/46] AMDGPU: Select region address space loads and stores
(#229500)
Fix the annoying asymmetry where we handle local and region for some
atomics but not the full suite of operations. I run into this about once per
year and want to be done with it.
The ugly bits here are to deal with the fact that it appears m0 still requires
initialization in gfx9+. It was simpler to handle the true16 case by
preprocessing the selection to a 32-bit extload as the load truly is, and go
through the ordinary m0 gluing.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 39 +-
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h | 1 +
llvm/lib/Target/AMDGPU/DSInstructions.td | 68 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 115 +-
llvm/test/CodeGen/AMDGPU/gds-atomic.ll | 16 +-
llvm/test/CodeGen/AMDGPU/gds-load-store.ll | 4568 +++++++++++++++++
6 files changed, 4737 insertions(+), 70 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/gds-load-store.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 86cb52485ee97e..f65668d0232b5e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -341,10 +341,35 @@ bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
return false;
}
-void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
- if (!Subtarget->d16PreservesUnusedBits())
- return;
+bool AMDGPUDAGToDAGISel::widenRegionLoad16(SDNode *N) const {
+ auto *Mem = cast<MemSDNode>(N);
+ EVT VT = N->getValueType(0);
+ if (Mem->getAddressSpace() != AMDGPUAS::REGION_ADDRESS || VT.isVector() ||
+ VT.getSizeInBits() != 16)
+ return false;
+
+ SDLoc SL(N);
+ auto *Ld = dyn_cast<LoadSDNode>(N);
+ ISD::LoadExtType ExtType =
+ Ld ? Ld->getExtensionType() : cast<AtomicSDNode>(N)->getExtensionType();
+ if (ExtType == ISD::NON_EXTLOAD)
+ ExtType = ISD::EXTLOAD;
+
+ SDValue NewLoad =
+ Ld ? CurDAG->getExtLoad(ExtType, SL, MVT::i32, Mem->getChain(),
+ Mem->getBasePtr(), Mem->getMemoryVT(),
+ Mem->getMemOperand())
+ : CurDAG->getAtomicLoad(ExtType, SL, Mem->getMemoryVT(), MVT::i32,
+ Mem->getChain(), Mem->getBasePtr(),
+ Mem->getMemOperand());
+
+ SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, SL, MVT::i16, NewLoad);
+ SDValue Ops[] = {CurDAG->getBitcast(VT, Trunc), NewLoad.getValue(1)};
+ CurDAG->ReplaceAllUsesWith(N, Ops);
+ return true;
+}
+void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
bool MadeChange = false;
@@ -356,7 +381,13 @@ void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
switch (N->getOpcode()) {
case ISD::BUILD_VECTOR:
// TODO: Match load d16 from shl (extload:i16), 16
- MadeChange |= matchLoadD16FromBuildVector(N);
+ if (Subtarget->d16PreservesUnusedBits())
+ MadeChange |= matchLoadD16FromBuildVector(N);
+ break;
+ case ISD::LOAD:
+ case ISD::ATOMIC_LOAD:
+ if (Subtarget->useRealTrue16Insts())
+ MadeChange |= widenRegionLoad16(N);
break;
default:
break;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
index a0e6b547b90af0..206fcf5064d91f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
@@ -67,6 +67,7 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
bool runOnMachineFunction(MachineFunction &MF) override;
bool matchLoadD16FromBuildVector(SDNode *N) const;
+ bool widenRegionLoad16(SDNode *N) const;
void PreprocessISelDAG() override;
void Select(SDNode *N) override;
void PostprocessISelDAG() override;
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index 0025672a69ebf4..6e6d9597e36c99 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -958,6 +958,11 @@ multiclass DSReadPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
let OtherPredicates = [NotLDSRequiresM0Init] in {
def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
}
+
+ let OtherPredicates = [HasGDS] in {
+ def : DSReadPat<inst, vt, !cast<PatFrag>(!subst("_local", "_region", frag)#"_m0"),
+ /*gds=*/1>;
+ }
}
multiclass DSReadPat_t16<DS_Pseudo inst, ValueType vt, string frag> {
@@ -975,6 +980,11 @@ multiclass DSReadPat_t16<DS_Pseudo inst, ValueType vt, string frag> {
let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = UseTrue16WithSramECC in {
def : DSReadPat_t16<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
}
+
+ let OtherPredicates = [HasGDS], True16Predicate = NotUseRealTrue16Insts in {
+ def : DSReadPat<inst, vt, !cast<PatFrag>(!subst("_local", "_region", frag)#"_m0"),
+ /*gds=*/1>;
+ }
}
class DSReadPat_D16 <DS_Pseudo inst, PatFrag frag, ValueType vt> : GCNPat <
@@ -1035,6 +1045,12 @@ class DSWritePat <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPa
(inst $ptr, getVregSrcForVT<vt>.ret:$value, Offset:$offset, (i1 gds))
>;
+class DSWritePat_t16 <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <
+ (frag vt:$value, (DS1Addr1Offset i32:$ptr, i32:$offset)),
+ (inst $ptr, (REG_SEQUENCE VGPR_32, $value, lo16, (i16 (IMPLICIT_DEF)), hi16),
+ Offset:$offset, (i1 gds))
+>;
+
multiclass DSWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
let OtherPredicates = [LDSRequiresM0Init] in {
def : DSWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
@@ -1043,6 +1059,11 @@ multiclass DSWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
let OtherPredicates = [NotLDSRequiresM0Init] in {
def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
}
+
+ let OtherPredicates = [HasGDS] in {
+ def : DSWritePat<inst, vt, !cast<PatFrag>(!subst("_local", "_region", frag)#"_m0"),
+ /*gds=*/1>;
+ }
}
multiclass DSWritePat_t16 <DS_Pseudo inst, ValueType vt, string frag> {
@@ -1058,6 +1079,16 @@ multiclass DSWritePat_t16 <DS_Pseudo inst, ValueType vt, string frag> {
def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>;
}
}
+
+ defvar region_frag = !cast<PatFrag>(!subst("_local", "_region", frag)#"_m0");
+ let OtherPredicates = [HasGDS] in {
+ let True16Predicate = NotUseRealTrue16Insts in {
+ def : DSWritePat<inst, vt, region_frag, /*gds=*/1>;
+ }
+ let True16Predicate = UseRealTrue16Insts in {
+ def : DSWritePat_t16<inst, vt, region_frag, /*gds=*/1>;
+ }
+ }
}
defm : DSWritePat_mc <DS_WRITE_B8, i32, "truncstorei8_local">;
@@ -1084,29 +1115,29 @@ def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>;
def : DSWritePat <DS_WRITE_B8_D16_HI, i32, truncstorei8_hi16_local>;
}
-class DS64Bit4ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
+class DS64Bit4ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <
(vt:$value (frag (DS64Bit4ByteAligned i32:$ptr, i32:$offset0, i32:$offset1))),
- (inst $ptr, $offset0, $offset1, (i1 0))
+ (inst $ptr, $offset0, $offset1, (i1 gds))
>;
// TODO: Should this use AVLdSt_64 for the class?
-class DS64Bit4ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat<
+class DS64Bit4ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat<
(frag vt:$value, (DS64Bit4ByteAligned i32:$ptr, i32:$offset0, i32:$offset1)),
(inst $ptr, (i32 (EXTRACT_SUBREG VReg_64:$value, sub0)),
(i32 (EXTRACT_SUBREG VReg_64:$value, sub1)), $offset0, $offset1,
- (i1 0))
+ (i1 gds))
>;
-class DS128Bit8ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
+class DS128Bit8ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <
(vt:$value (frag (DS128Bit8ByteAligned i32:$ptr, i32:$offset0, i32:$offset1))),
- (inst $ptr, $offset0, $offset1, (i1 0))
+ (inst $ptr, $offset0, $offset1, (i1 gds))
>;
-class DS128Bit8ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat<
+class DS128Bit8ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat<
(frag vt:$value, (DS128Bit8ByteAligned i32:$ptr, i32:$offset0, i32:$offset1)),
(inst $ptr, (i64 (EXTRACT_SUBREG VReg_128:$value, sub0_sub1)),
(i64 (EXTRACT_SUBREG VReg_128:$value, sub2_sub3)), $offset0, $offset1,
- (i1 0))
+ (i1 gds))
>;
multiclass DS64Bit4ByteAlignedPat_mc<ValueType vt> {
@@ -1115,6 +1146,11 @@ multiclass DS64Bit4ByteAlignedPat_mc<ValueType vt> {
def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32, vt, store_local_m0>;
}
+ let SubtargetPredicate = HasUsableDSOffset, OtherPredicates = [HasGDS] in {
+ def : DS64Bit4ByteAlignedReadPat<DS_READ2_B32, vt, load_region_m0, /*gds=*/1>;
+ def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32, vt, store_region_m0, /*gds=*/1>;
+ }
+
let OtherPredicates = [NotLDSRequiresM0Init, NotNeedsAligned2addrDS] in {
def : DS64Bit4ByteAlignedReadPat<DS_READ2_B32_gfx9, vt, load_local>;
def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32_gfx9, vt, store_local>;
@@ -1132,6 +1168,11 @@ multiclass DS128Bit8ByteAlignedPat_mc<ValueType vt> {
def : DS128Bit8ByteAlignedWritePat<DS_WRITE2_B64, vt, store_local_m0>;
}
+ let SubtargetPredicate = HasUsableDSOffset, OtherPredicates = [HasGDS] in {
+ def : DS128Bit8ByteAlignedReadPat<DS_READ2_B64, vt, load_region_m0, /*gds=*/1>;
+ def : DS128Bit8ByteAlignedWritePat<DS_WRITE2_B64, vt, store_region_m0, /*gds=*/1>;
+ }
+
let OtherPredicates = [NotLDSRequiresM0Init, NotNeedsAligned2addrDS] in {
def : DS128Bit8ByteAlignedReadPat<DS_READ2_B64_gfx9, vt, load_local>;
def : DS128Bit8ByteAlignedWritePat<DS_WRITE2_B64_gfx9, vt, store_local>;
@@ -1543,6 +1584,12 @@ multiclass DS_Real_gfx11<bits<8> op, DS_Pseudo ps = !cast<DS_Pseudo>(NAME#"_gfx9
def _gfx11 :
Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12_gfx13<op, ps, SIEncodingFamily.GFX11,
name>;
+ if !eq(!cast<string>(ps), NAME#"_gfx9") then {
+ let isCodeGenOnly = 1 in
+ def _m0_gfx11 :
+ Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12_gfx13<op,
+ !cast<DS_Pseudo>(NAME), SIEncodingFamily.GFX11, name>;
+ }
if !ne(ps.Mnemonic, name) then
def : AMDGPUMnemonicAlias<ps.Mnemonic, name>;
} // End AssemblerPredicate
@@ -1626,6 +1673,11 @@ let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in {
multiclass DS_Real_gfx10<bits<8> op, DS_Pseudo ps = !cast<DS_Pseudo>(NAME)> {
def _gfx10 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12_gfx13<op,
ps, SIEncodingFamily.GFX10>;
+ if !eq(!cast<string>(ps), NAME#"_gfx9") then {
+ let isCodeGenOnly = 1 in
+ def _m0_gfx10 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12_gfx13<op,
+ !cast<DS_Pseudo>(NAME), SIEncodingFamily.GFX10>;
+ }
}
} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10"
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 1c42c49f6efdb8..e7425708d51a00 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -559,53 +559,55 @@ def sextloadi16_glue : PatFrag<(ops node:$ptr), (sextload_glue node:$ptr)> {
}
-let IsLoad = 1, AddressSpaces = LoadAddress_local.AddrSpaces in {
-def load_local_m0 : PatFrag<(ops node:$ptr), (load_glue node:$ptr)> {
+foreach as = [ "local", "region" ] in {
+let IsLoad = 1, AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
+def load_#as#_m0 : PatFrag<(ops node:$ptr), (load_glue node:$ptr)> {
let IsNonExtLoad = 1;
}
-def extloadi8_local_m0 : PatFrag<(ops node:$ptr), (extloadi8_glue node:$ptr)>;
-def sextloadi8_local_m0 : PatFrag<(ops node:$ptr), (sextloadi8_glue node:$ptr)>;
-def zextloadi8_local_m0 : PatFrag<(ops node:$ptr), (zextloadi8_glue node:$ptr)>;
+def extloadi8_#as#_m0 : PatFrag<(ops node:$ptr), (extloadi8_glue node:$ptr)>;
+def sextloadi8_#as#_m0 : PatFrag<(ops node:$ptr), (sextloadi8_glue node:$ptr)>;
+def zextloadi8_#as#_m0 : PatFrag<(ops node:$ptr), (zextloadi8_glue node:$ptr)>;
-def extloadi16_local_m0 : PatFrag<(ops node:$ptr), (extloadi16_glue node:$ptr)>;
-def sextloadi16_local_m0 : PatFrag<(ops node:$ptr), (sextloadi16_glue node:$ptr)>;
-def zextloadi16_local_m0 : PatFrag<(ops node:$ptr), (zextloadi16_glue node:$ptr)>;
-} // End IsLoad = 1, , AddressSpaces = LoadAddress_local.AddrSpaces
+def extloadi16_#as#_m0 : PatFrag<(ops node:$ptr), (extloadi16_glue node:$ptr)>;
+def sextloadi16_#as#_m0 : PatFrag<(ops node:$ptr), (sextloadi16_glue node:$ptr)>;
+def zextloadi16_#as#_m0 : PatFrag<(ops node:$ptr), (zextloadi16_glue node:$ptr)>;
+} // End IsLoad = 1, AddressSpaces = LoadAddress_#as.AddrSpaces
-def load_align8_local_m0 : PatFrag<(ops node:$ptr),
- (load_local_m0 node:$ptr)> {
+def load_align8_#as#_m0 : PatFrag<(ops node:$ptr),
+ (!cast<PatFrag>("load_"#as#"_m0") node:$ptr)> {
let IsLoad = 1;
int MinAlignment = 8;
}
-def load_align16_local_m0 : PatFrag<(ops node:$ptr),
- (load_local_m0 node:$ptr)> {
+def load_align16_#as#_m0 : PatFrag<(ops node:$ptr),
+ (!cast<PatFrag>("load_"#as#"_m0") node:$ptr)> {
let IsLoad = 1;
int MinAlignment = 16;
}
-let IsAtomic = 1, AddressSpaces = LoadAddress_local.AddrSpaces in {
-def atomic_load_nonext_16_local_m0 : PatFrag<(ops node:$ptr),
+let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
+def atomic_load_nonext_16_#as#_m0 : PatFrag<(ops node:$ptr),
(atomic_load_nonext_16_glue node:$ptr)>;
-def atomic_load_nonext_32_local_m0 : PatFrag<(ops node:$ptr),
+def atomic_load_nonext_32_#as#_m0 : PatFrag<(ops node:$ptr),
(atomic_load_nonext_32_glue node:$ptr)>;
-def atomic_load_nonext_64_local_m0 : PatFrag<(ops node:$ptr),
+def atomic_load_nonext_64_#as#_m0 : PatFrag<(ops node:$ptr),
(atomic_load_nonext_64_glue node:$ptr)>;
-def atomic_load_zext_8_local_m0 : PatFrag<(ops node:$ptr),
+def atomic_load_zext_8_#as#_m0 : PatFrag<(ops node:$ptr),
(atomic_load_zext_8_glue node:$ptr)>;
-def atomic_load_sext_8_local_m0 : PatFrag<(ops node:$ptr),
+def atomic_load_sext_8_#as#_m0 : PatFrag<(ops node:$ptr),
(atomic_load_sext_8_glue node:$ptr)>;
-def atomic_load_aext_8_local_m0 : PatFrag<(ops node:$ptr),
+def atomic_load_aext_8_#as#_m0 : PatFrag<(ops node:$ptr),
(atomic_load_aext_8_glue node:$ptr)>;
-def atomic_load_zext_16_local_m0 : PatFrag<(ops node:$ptr),
+def atomic_load_zext_16_#as#_m0 : PatFrag<(ops node:$ptr),
(atomic_load_zext_16_glue node:$ptr)>;
-def atomic_load_sext_16_local_m0 : PatFrag<(ops node:$ptr),
+def atomic_load_sext_16_#as#_m0 : PatFrag<(ops node:$ptr),
(atomic_load_sext_16_glue node:$ptr)>;
-def atomic_load_aext_16_local_m0 : PatFrag<(ops node:$ptr),
+def atomic_load_aext_16_#as#_m0 : PatFrag<(ops node:$ptr),
(atomic_load_aext_16_glue node:$ptr)>;
-} // End let AddressSpaces = LoadAddress_local.AddrSpaces
+} // End let IsAtomic = 1, AddressSpaces = LoadAddress_#as.AddrSpaces
+} // End foreach as
def AMDGPUst_glue : SDNode <"ISD::STORE", SDTStore,
@@ -646,26 +648,28 @@ def truncstorei16_glue : PatFrag<(ops node:$val, node:$ptr),
let MemoryVT = i16;
}
-let IsStore = 1, AddressSpaces = StoreAddress_local.AddrSpaces in {
-def store_local_m0 : PatFrag<(ops node:$val, node:$ptr),
- (store_glue node:$val, node:$ptr)>;
-def truncstorei8_local_m0 : PatFrag<(ops node:$val, node:$ptr),
- (truncstorei8_glue node:$val, node:$ptr)>;
-def truncstorei16_local_m0 : PatFrag<(ops node:$val, node:$ptr),
+foreach as = [ "local", "region" ] in {
+let IsStore = 1, AddressSpaces = !cast<AddressSpaceList>("StoreAddress_"#as).AddrSpaces in {
+def store_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
+ (store_glue node:$val, node:$ptr)>;
+def truncstorei8_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
+ (truncstorei8_glue node:$val, node:$ptr)>;
+def truncstorei16_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
(truncstorei16_glue node:$val, node:$ptr)>;
}
-def store_align8_local_m0 : PatFrag <(ops node:$value, node:$ptr),
- (store_local_m0 node:$value, node:$ptr)>,
- Aligned<8> {
+def store_align8_#as#_m0 : PatFrag <(ops node:$value, node:$ptr),
+ (!cast<PatFrag>("store_"#as#"_m0") node:$value, node:$ptr)>,
+ Aligned<8> {
let IsStore = 1;
}
-def store_align16_local_m0 : PatFrag <(ops node:$value, node:$ptr),
- (store_local_m0 node:$value, node:$ptr)>,
+def store_align16_#as#_m0 : PatFrag <(ops node:$value, node:$ptr),
+ (!cast<PatFrag>("store_"#as#"_m0") node:$value, node:$ptr)>,
Aligned<16> {
let IsStore = 1;
}
+} // End foreach as
let PredicateCode = [{return cast<MemSDNode>(N)->getAlign() < 4;}],
GISelPredicateCode = [{return (*MI.memoperands_begin())->getAlign() < 4;}],
@@ -676,20 +680,26 @@ def load_align_less_than_4_local : PatFrag<(ops node:$ptr),
let IsNonExtLoad = 1;
}
-def load_align_less_than_4_local_m0 : PatFrag<(ops node:$ptr),
- (load_local_m0 node:$ptr)> {
- let IsLoad = 1;
- let IsNonExtLoad = 1;
-}
def store_align_less_than_4_local : PatFrag <(ops node:$value, node:$ptr),
(store_local node:$value, node:$ptr)> {
let IsStore = 1;
let IsTruncStore = 0;
}
+}
+
+foreach as = [ "local", "region" ] in
+let PredicateCode = [{return cast<MemSDNode>(N)->getAlign() < 4;}],
+ GISelPredicateCode = [{return (*MI.memoperands_begin())->getAlign() < 4;}],
+ AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
+def load_align_less_than_4_#as#_m0 : PatFrag<(ops node:$ptr),
+ (!cast<PatFrag>("load_"#as#"_m0") node:$ptr)> {
+ let IsLoad = 1;
+ let IsNonExtLoad = 1;
+}
-def store_align_less_than_4_local_m0 : PatFrag <(ops node:$value, node:$ptr),
- (store_local_m0 node:$value, node:$ptr)> {
+def store_align_less_than_4_#as#_m0 : PatFrag <(ops node:$value, node:$ptr),
+ (!cast<PatFrag>("store_"#as#"_m0") node:$value, node:$ptr)> {
let IsStore = 1;
let IsTruncStore = 0;
}
@@ -723,16 +733,17 @@ def atomic_store_64_glue : PatFrag <
let MemoryVT = i64;
}
-let IsAtomic = 1, AddressSpaces = StoreAddress_local.AddrSpaces in {
-def atomic_store_8_local_m0 : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_8_glue node:$val, node:$ptr)>;
-def atomic_store_16_local_m0 : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_16_glue node:$val, node:$ptr)>;
-def atomic_store_32_local_m0 : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_32_glue node:$val, node:$ptr)>;
-def atomic_store_64_local_m0 : PatFrag<(ops node:$val, node:$ptr),
- (atomic_store_64_glue node:$val, node:$ptr)>;
-} // End let IsAtomic = 1, AddressSpaces = StoreAddress_local.AddrSpaces
+foreach as = [ "local", "region" ] in
+let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("StoreAddress_"#as).AddrSpaces in {
+def atomic_store_8_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_8_glue node:$val, node:$ptr)>;
+def atomic_store_16_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_16_glue node:$val, node:$ptr)>;
+def atomic_store_32_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_32_glue node:$val, node:$ptr)>;
+def atomic_store_64_#as#_m0 : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_64_glue node:$val, node:$ptr)>;
+} // End let IsAtomic = 1, AddressSpaces = StoreAddress_#as.AddrSpaces
//===----------------------------------------------------------------------===//
diff --git a/llvm/test/CodeGen/AMDGPU/gds-atomic.ll b/llvm/test/CodeGen/AMDGPU/gds-atomic.ll
index cac871203191e5..ad507245a126b7 100644
--- a/llvm/test/CodeGen/AMDGPU/gds-atomic.ll
+++ b/llvm/test/CodeGen/AMDGPU/gds-atomic.ll
@@ -1,7 +1,9 @@
-; RUN: llc -mtriple=amdgpu7.04 < %s | FileCheck -check-prefixes=GCN,FUNC %s
-; RUN: llc -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,FUNC %s
-; RUN: llc -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,FUNC %s
-; RUN: llc -mtriple=amdgpu9 --amdhsa-code-object-version=6 < %s | FileCheck -check-prefixes=GCN,FUNC %s
+; RUN: llc -mtriple=amdgpu7.04 < %s | FileCheck -check-prefixes=GCN,PREGFX11,FUNC %s
+; RUN: llc -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,PREGFX11,FUNC %s
+; RUN: llc -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,PREGFX11,FUNC %s
+; RUN: llc -mtriple=amdgpu9 --amdhsa-code-object-version=6 < %s | FileCheck -check-prefixes=GCN,PREGFX11,FUNC %s
+; RUN: llc -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GCN,PREGFX11,FUNC %s
+; RUN: llc -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=GCN,GFX11,FUNC %s
; FUNC-LABEL: {{^}}atomic_add_ret_gds:
; GCN-DAG: v_mov_b32_e32 v[[OFF:[0-9]+]], s
@@ -106,7 +108,8 @@ define amdgpu_kernel void @atomic_imax_ret_gds(ptr addrspace(1) %out, ptr addrsp
; FUNC-LABEL: {{^}}atomic_xchg_ret_gds:
; GCN-DAG: v_mov_b32_e32 v[[OFF:[0-9]+]], s
; GCN-DAG: s_movk_i32 m0, 0x1000
-; GCN: ds_wrxchg_rtn_b32 v{{[0-9]+}}, v[[OFF]], v{{[0-9]+}} gds
+; PREGFX11: ds_wrxchg_rtn_b32 v{{[0-9]+}}, v[[OFF]], v{{[0-9]+}} gds
+; GFX11: ds_storexchg_rtn_b32 v{{[0-9]+}}, v[[OFF]], v{{[0-9]+}} gds
define amdgpu_kernel void @atomic_xchg_ret_gds(ptr addrspace(1) %out, ptr addrspace(2) %gds) #1 {
%val = atomicrmw xchg ptr addrspace(2) %gds, i32 5 acq_rel
store i32 %val, ptr addrspace(1) %out
@@ -116,7 +119,8 @@ define amdgpu_kernel void @atomic_xchg_ret_gds(ptr addrspace(1) %out, ptr addrsp
; FUNC-LABEL: {{^}}atomic_cmpxchg_ret_gds:
; GCN-DAG: v_mov_b32_e32 v[[OFF:[0-9]+]], s
; GCN-DAG: s_movk_i32 m0, 0x1000
-; GCN: ds_cmpst_rtn_b32 v{{[0-9]+}}, v[[OFF:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} gds
+; PREGFX11: ds_cmpst_rtn_b32 v{{[0-9]+}}, v[[OFF]], v{{[0-9]+}}, v{{[0-9]+}} gds
+; GFX11: ds_cmpstore_rtn_b32 v{{[0-9]+}}, v[[OFF]], v{{[0-9]+}}, v{{[0-9]+}} gds
define amdgpu_kernel void @atomic_cmpxchg_ret_gds(ptr addrspace(1) %out, ptr addrspace(2) %gds) #1 {
%val = cmpxchg ptr addrspace(2) %gds, i32 0, i32 1 acquire acquire
%x = extractvalue { i32, i1 } %val, 0
diff --git a/llvm/test/CodeGen/AMDGPU/gds-load-store.ll b/llvm/test/CodeGen/AMDGPU/gds-load-store.ll
new file mode 100644
index 00000000000000..f47811c8ee4e90
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/gds-load-store.ll
@@ -0,0 +1,4568 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu6.00 < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -mtriple=amdgpu7.04 < %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgpu10.30 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+
+define i32 @load_i32(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_i32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_i32:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b32 v0, v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b32 v0, v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b32 v0, v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b32 v0, v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load i32, ptr addrspace(2) %ptr
+ ret i32 %val
+}
+
+define i32 @load_i32_offset(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_i32_offset:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 64, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_i32_offset:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b32 v0, v0 offset:64 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_i32_offset:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b32 v0, v0 offset:64 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_i32_offset:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b32 v0, v0 offset:64 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_i32_offset:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b32 v0, v0 offset:64 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr i32, ptr addrspace(2) %ptr, i32 16
+ %val = load i32, ptr addrspace(2) %gep
+ ret i32 %val
+}
+
+define i32 @sextload_i8(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: sextload_i8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_i8 v0, v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: sextload_i8:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_i8 v0, v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: sextload_i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_i8 v0, v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: sextload_i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_i8 v0, v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: sextload_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_i8 v0, v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load i8, ptr addrspace(2) %ptr
+ %ext = sext i8 %val to i32
+ ret i32 %ext
+}
+
+define i32 @zextload_i8(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: zextload_i8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_u8 v0, v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: zextload_i8:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_u8 v0, v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: zextload_i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_u8 v0, v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: zextload_i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_u8 v0, v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: zextload_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_u8 v0, v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load i8, ptr addrspace(2) %ptr
+ %ext = zext i8 %val to i32
+ ret i32 %ext
+}
+
+define i32 @sextload_i16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: sextload_i16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_i16 v0, v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: sextload_i16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_i16 v0, v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: sextload_i16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_i16 v0, v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: sextload_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_i16 v0, v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: sextload_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_i16 v0, v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load i16, ptr addrspace(2) %ptr
+ %ext = sext i16 %val to i32
+ ret i32 %ext
+}
+
+define i32 @zextload_i16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: zextload_i16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_u16 v0, v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: zextload_i16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_u16 v0, v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: zextload_i16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_u16 v0, v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: zextload_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_u16 v0, v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: zextload_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_u16 v0, v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load i16, ptr addrspace(2) %ptr
+ %ext = zext i16 %val to i32
+ ret i32 %ext
+}
+
+define i8 @load_i8(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_i8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_u8 v0, v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_i8:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_u8 v0, v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_u8 v0, v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_u8 v0, v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_u8 v0, v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load i8, ptr addrspace(2) %ptr
+ ret i8 %val
+}
+
+define half @load_f16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_u16 v0, v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_u16 v0, v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_u16 v0, v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_u16 v0, v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_u16 v0, v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load half, ptr addrspace(2) %ptr
+ ret half %val
+}
+
+define i64 @load_i64_align4(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_i64_align4:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 4, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: ds_read_b32 v1, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_i64_align4:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_i64_align4:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_i64_align4:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_i64_align4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_2addr_b32 v[0:1], v0 offset1:1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load i64, ptr addrspace(2) %ptr, align 4
+ ret i64 %val
+}
+
+define i64 @load_i64_align8(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_i64_align8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_i64_align8:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_i64_align8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_i64_align8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_i64_align8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b64 v[0:1], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load i64, ptr addrspace(2) %ptr, align 8
+ ret i64 %val
+}
+
+define <3 x i32> @load_v3i32_align16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v3i32_align16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: ds_read_b32 v2, v2 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v3i32_align16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b96 v[0:2], v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v3i32_align16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b96 v[0:2], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v3i32_align16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b96 v[0:2], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v3i32_align16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b96 v[0:2], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <3 x i32>, ptr addrspace(2) %ptr, align 16
+ ret <3 x i32> %val
+}
+
+define <4 x i32> @load_v4i32_align8(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v4i32_align8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: ds_read_b64 v[2:3], v2 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v4i32_align8:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read2_b64 v[0:3], v0 offset1:1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v4i32_align8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read2_b64 v[0:3], v0 offset1:1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v4i32_align8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read2_b64 v[0:3], v0 offset1:1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v4i32_align8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_2addr_b64 v[0:3], v0 offset1:1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <4 x i32>, ptr addrspace(2) %ptr, align 8
+ ret <4 x i32> %val
+}
+
+define <4 x i32> @load_v4i32_align16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v4i32_align16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: ds_read_b64 v[2:3], v2 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v4i32_align16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v4i32_align16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v4i32_align16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v4i32_align16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b128 v[0:3], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <4 x i32>, ptr addrspace(2) %ptr, align 16
+ ret <4 x i32> %val
+}
+
+define void @store_i32(ptr addrspace(2) %ptr, i32 %val) {
+; GFX6-LABEL: store_i32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_i32:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b32 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b32 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b32 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b32 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store i32 %val, ptr addrspace(2) %ptr
+ ret void
+}
+
+define void @store_i32_offset(ptr addrspace(2) %ptr, i32 %val) {
+; GFX6-LABEL: store_i32_offset:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 64, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_i32_offset:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b32 v0, v1 offset:64 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_i32_offset:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b32 v0, v1 offset:64 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_i32_offset:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b32 v0, v1 offset:64 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_i32_offset:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b32 v0, v1 offset:64 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr i32, ptr addrspace(2) %ptr, i32 16
+ store i32 %val, ptr addrspace(2) %gep
+ ret void
+}
+
+define void @truncstore_i32_to_i8(ptr addrspace(2) %ptr, i32 %val) {
+; GFX6-LABEL: truncstore_i32_to_i8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b8 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: truncstore_i32_to_i8:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b8 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: truncstore_i32_to_i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b8 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: truncstore_i32_to_i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b8 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: truncstore_i32_to_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b8 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %trunc = trunc i32 %val to i8
+ store i8 %trunc, ptr addrspace(2) %ptr
+ ret void
+}
+
+define void @truncstore_i32_to_i16(ptr addrspace(2) %ptr, i32 %val) {
+; GFX6-LABEL: truncstore_i32_to_i16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b16 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: truncstore_i32_to_i16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b16 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: truncstore_i32_to_i16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b16 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: truncstore_i32_to_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b16 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: truncstore_i32_to_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b16 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %trunc = trunc i32 %val to i16
+ store i16 %trunc, ptr addrspace(2) %ptr
+ ret void
+}
+
+define void @store_i8(ptr addrspace(2) %ptr, i8 %val) {
+; GFX6-LABEL: store_i8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b8 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_i8:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b8 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b8 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b8 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b8 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store i8 %val, ptr addrspace(2) %ptr
+ ret void
+}
+
+define void @store_f16(ptr addrspace(2) %ptr, half %val) {
+; GFX6-LABEL: store_f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b16 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b16 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b16 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b16 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b16 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store half %val, ptr addrspace(2) %ptr
+ ret void
+}
+
+define void @store_i64_align4(ptr addrspace(2) %ptr, i64 %val) {
+; GFX6-LABEL: store_i64_align4:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 4, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v3, v2 gds
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_i64_align4:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_i64_align4:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_i64_align4:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_i64_align4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_2addr_b32 v0, v1, v2 offset1:1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store i64 %val, ptr addrspace(2) %ptr, align 4
+ ret void
+}
+
+define void @store_i64_align8(ptr addrspace(2) %ptr, i64 %val) {
+; GFX6-LABEL: store_i64_align8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_i64_align8:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_i64_align8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_i64_align8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_i64_align8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b64 v0, v[1:2] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store i64 %val, ptr addrspace(2) %ptr, align 8
+ ret void
+}
+
+define void @store_v3i32_align16(ptr addrspace(2) %ptr, <3 x i32> %val) {
+; GFX6-LABEL: store_v3i32_align16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 8, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v4, v3 gds
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v3i32_align16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b96 v0, v[1:3] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v3i32_align16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b96 v0, v[1:3] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v3i32_align16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b96 v0, v[1:3] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v3i32_align16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b96 v0, v[1:3] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <3 x i32> %val, ptr addrspace(2) %ptr, align 16
+ ret void
+}
+
+define void @store_v4i32_align8(ptr addrspace(2) %ptr, <4 x i32> %val) {
+; GFX6-LABEL: store_v4i32_align8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 8, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v5, v[3:4] gds
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v4i32_align8:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write2_b64 v0, v[1:2], v[3:4] offset1:1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v4i32_align8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write2_b64 v0, v[1:2], v[3:4] offset1:1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v4i32_align8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write2_b64 v0, v[1:2], v[3:4] offset1:1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v4i32_align8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_2addr_b64 v0, v[1:2], v[3:4] offset1:1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <4 x i32> %val, ptr addrspace(2) %ptr, align 8
+ ret void
+}
+
+define void @store_v4i32_align16(ptr addrspace(2) %ptr, <4 x i32> %val) {
+; GFX6-LABEL: store_v4i32_align16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 8, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v5, v[3:4] gds
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v4i32_align16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v4i32_align16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v4i32_align16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v4i32_align16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b128 v0, v[1:4] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <4 x i32> %val, ptr addrspace(2) %ptr, align 16
+ ret void
+}
+
+define i32 @atomic_load_i32(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_i32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_i32:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b32 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b32 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b32 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_b32 v0, v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic i32, ptr addrspace(2) %ptr seq_cst, align 4
+ ret i32 %val
+}
+
+define i16 @atomic_load_i16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_i16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_u16 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_i16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_u16 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_i16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_u16 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_u16 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_u16 v0, v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic i16, ptr addrspace(2) %ptr seq_cst, align 2
+ ret i16 %val
+}
+
+define i64 @atomic_load_i64(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_i64:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_i64:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_i64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_b64 v[0:1], v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic i64, ptr addrspace(2) %ptr seq_cst, align 8
+ ret i64 %val
+}
+
+define void @atomic_store_i32(ptr addrspace(2) %ptr, i32 %val) {
+; GFX6-LABEL: atomic_store_i32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_i32:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b32 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b32 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b32 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b32 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic i32 %val, ptr addrspace(2) %ptr seq_cst, align 4
+ ret void
+}
+
+define void @atomic_store_i16(ptr addrspace(2) %ptr, i16 %val) {
+; GFX6-LABEL: atomic_store_i16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b16 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_i16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b16 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_i16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b16 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b16 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b16 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic i16 %val, ptr addrspace(2) %ptr seq_cst, align 2
+ ret void
+}
+
+define void @atomic_store_i64(ptr addrspace(2) %ptr, i64 %val) {
+; GFX6-LABEL: atomic_store_i64:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_i64:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_i64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b64 v0, v[1:2] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic i64 %val, ptr addrspace(2) %ptr seq_cst, align 8
+ ret void
+}
+define i8 @atomic_load_i8(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_i8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_u8 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_i8:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_u8 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_u8 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_u8 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_u8 v0, v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic i8, ptr addrspace(2) %ptr seq_cst, align 1
+ ret i8 %val
+}
+
+define i32 @atomic_load_zext_i8(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_zext_i8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_u8 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_zext_i8:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_u8 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_zext_i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_u8 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_zext_i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_u8 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: atomic_load_zext_i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 m0, 0
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: ds_load_u8 v0, v0 gds
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: s_waitcnt expcnt(0)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: atomic_load_zext_i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 m0, 0
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: ds_load_u8 v0, v0 gds
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: s_waitcnt expcnt(0)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+ %load = load atomic i8, ptr addrspace(2) %ptr seq_cst, align 1
+ %ext = zext i8 %load to i32
+ ret i32 %ext
+}
+
+define i32 @atomic_load_sext_i8(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_sext_i8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_u8 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_sext_i8:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_u8 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_sext_i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_u8 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_sext_i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_u8 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_sext_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_u8 v0, v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %load = load atomic i8, ptr addrspace(2) %ptr seq_cst, align 1
+ %ext = sext i8 %load to i32
+ ret i32 %ext
+}
+
+define i32 @atomic_load_zext_i16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_zext_i16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_u16 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_zext_i16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_u16 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_zext_i16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_u16 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_zext_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_u16 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: atomic_load_zext_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 m0, 0
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: ds_load_u16 v0, v0 gds
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: s_waitcnt expcnt(0)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: atomic_load_zext_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 m0, 0
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: ds_load_u16 v0, v0 gds
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: s_waitcnt expcnt(0)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+ %load = load atomic i16, ptr addrspace(2) %ptr seq_cst, align 2
+ %ext = zext i16 %load to i32
+ ret i32 %ext
+}
+
+define i32 @atomic_load_sext_i16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_sext_i16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_u16 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_sext_i16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_u16 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_sext_i16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_u16 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_sext_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_u16 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_sext_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_u16 v0, v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %load = load atomic i16, ptr addrspace(2) %ptr seq_cst, align 2
+ %ext = sext i16 %load to i32
+ ret i32 %ext
+}
+
+define i32 @atomic_load_i32_offset(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_i32_offset:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 64, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_i32_offset:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b32 v0, v0 offset:64 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_i32_offset:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b32 v0, v0 offset:64 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_i32_offset:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b32 v0, v0 offset:64 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_i32_offset:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_b32 v0, v0 offset:64 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr inbounds i32, ptr addrspace(2) %ptr, i32 16
+ %val = load atomic i32, ptr addrspace(2) %gep seq_cst, align 4
+ ret i32 %val
+}
+
+define float @atomic_load_f32(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_f32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_f32:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b32 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b32 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b32 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_b32 v0, v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic float, ptr addrspace(2) %ptr seq_cst, align 4
+ ret float %val
+}
+
+define double @atomic_load_f64(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_f64:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_f64:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_f64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_f64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_b64 v[0:1], v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic double, ptr addrspace(2) %ptr seq_cst, align 8
+ ret double %val
+}
+
+define <2 x i16> @atomic_load_v2i16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_v2i16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_v2i16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b32 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_v2i16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b32 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_v2i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b32 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_v2i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_b32 v0, v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic <2 x i16>, ptr addrspace(2) %ptr seq_cst, align 4
+ ret <2 x i16> %val
+}
+
+define ptr addrspace(1) @atomic_load_p1(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_p1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_p1:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_p1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_p1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_p1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_b64 v[0:1], v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic ptr addrspace(1), ptr addrspace(2) %ptr seq_cst, align 8
+ ret ptr addrspace(1) %val
+}
+
+define ptr addrspace(2) @atomic_load_p2(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_p2:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_p2:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b32 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_p2:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b32 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_p2:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b32 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_p2:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_b32 v0, v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic ptr addrspace(2), ptr addrspace(2) %ptr seq_cst, align 4
+ ret ptr addrspace(2) %val
+}
+
+define ptr addrspace(3) @atomic_load_p3(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_p3:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_p3:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b32 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_p3:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b32 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_p3:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b32 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_p3:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_b32 v0, v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic ptr addrspace(3), ptr addrspace(2) %ptr seq_cst, align 4
+ ret ptr addrspace(3) %val
+}
+
+define void @atomic_store_i8(ptr addrspace(2) %ptr, i8 %val) {
+; GFX6-LABEL: atomic_store_i8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b8 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_i8:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b8 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b8 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b8 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b8 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic i8 %val, ptr addrspace(2) %ptr seq_cst, align 1
+ ret void
+}
+
+define void @atomic_store_i32_offset(ptr addrspace(2) %ptr, i32 %val) {
+; GFX6-LABEL: atomic_store_i32_offset:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 64, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_i32_offset:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b32 v0, v1 offset:64 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_i32_offset:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b32 v0, v1 offset:64 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_i32_offset:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b32 v0, v1 offset:64 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_i32_offset:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b32 v0, v1 offset:64 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr inbounds i32, ptr addrspace(2) %ptr, i32 16
+ store atomic i32 %val, ptr addrspace(2) %gep seq_cst, align 4
+ ret void
+}
+
+define void @atomic_store_f32(ptr addrspace(2) %ptr, float %val) {
+; GFX6-LABEL: atomic_store_f32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_f32:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b32 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b32 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b32 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b32 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic float %val, ptr addrspace(2) %ptr seq_cst, align 4
+ ret void
+}
+
+define void @atomic_store_f64(ptr addrspace(2) %ptr, double %val) {
+; GFX6-LABEL: atomic_store_f64:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_f64:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_f64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_f64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b64 v0, v[1:2] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic double %val, ptr addrspace(2) %ptr seq_cst, align 8
+ ret void
+}
+
+define void @atomic_store_v2i16(ptr addrspace(2) %ptr, <2 x i16> %val) {
+; GFX6-LABEL: atomic_store_v2i16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_v2i16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b32 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_v2i16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b32 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_v2i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b32 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_v2i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b32 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic <2 x i16> %val, ptr addrspace(2) %ptr seq_cst, align 4
+ ret void
+}
+
+define void @atomic_store_p1(ptr addrspace(2) %ptr, ptr addrspace(1) %val) {
+; GFX6-LABEL: atomic_store_p1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_p1:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_p1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_p1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_p1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b64 v0, v[1:2] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic ptr addrspace(1) %val, ptr addrspace(2) %ptr seq_cst, align 8
+ ret void
+}
+
+define void @atomic_store_p2(ptr addrspace(2) %ptr, ptr addrspace(2) %val) {
+; GFX6-LABEL: atomic_store_p2:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_p2:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b32 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_p2:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b32 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_p2:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b32 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_p2:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b32 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic ptr addrspace(2) %val, ptr addrspace(2) %ptr seq_cst, align 4
+ ret void
+}
+
+define void @atomic_store_p3(ptr addrspace(2) %ptr, ptr addrspace(3) %val) {
+; GFX6-LABEL: atomic_store_p3:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_p3:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b32 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_p3:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b32 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_p3:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b32 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_p3:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b32 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic ptr addrspace(3) %val, ptr addrspace(2) %ptr seq_cst, align 4
+ ret void
+}
+
+define ptr addrspace(1) @load_p1(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_p1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_p1:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_p1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_p1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_p1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b64 v[0:1], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load ptr addrspace(1), ptr addrspace(2) %ptr, align 8
+ ret ptr addrspace(1) %val
+}
+
+define ptr addrspace(2) @load_p2(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_p2:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_p2:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b32 v0, v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_p2:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b32 v0, v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_p2:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b32 v0, v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_p2:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b32 v0, v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load ptr addrspace(2), ptr addrspace(2) %ptr, align 4
+ ret ptr addrspace(2) %val
+}
+
+define ptr addrspace(3) @load_p3(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_p3:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_p3:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b32 v0, v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_p3:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b32 v0, v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_p3:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b32 v0, v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_p3:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b32 v0, v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load ptr addrspace(3), ptr addrspace(2) %ptr, align 4
+ ret ptr addrspace(3) %val
+}
+
+define <2 x ptr addrspace(1)> @load_v2p1(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v2p1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: ds_read_b64 v[2:3], v2 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v2p1:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v2p1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v2p1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v2p1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b128 v[0:3], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <2 x ptr addrspace(1)>, ptr addrspace(2) %ptr, align 16
+ ret <2 x ptr addrspace(1)> %val
+}
+
+define <2 x ptr addrspace(2)> @load_v2p2(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v2p2:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v2p2:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v2p2:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v2p2:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v2p2:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b64 v[0:1], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <2 x ptr addrspace(2)>, ptr addrspace(2) %ptr, align 8
+ ret <2 x ptr addrspace(2)> %val
+}
+
+define <2 x ptr addrspace(3)> @load_v2p3(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v2p3:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v2p3:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v2p3:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v2p3:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v2p3:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b64 v[0:1], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <2 x ptr addrspace(3)>, ptr addrspace(2) %ptr, align 8
+ ret <2 x ptr addrspace(3)> %val
+}
+
+define <3 x ptr addrspace(3)> @load_v3p3(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v3p3:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: ds_read_b32 v2, v2 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v3p3:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b96 v[0:2], v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v3p3:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b96 v[0:2], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v3p3:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b96 v[0:2], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v3p3:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b96 v[0:2], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <3 x ptr addrspace(3)>, ptr addrspace(2) %ptr, align 16
+ ret <3 x ptr addrspace(3)> %val
+}
+
+define <4 x ptr addrspace(3)> @load_v4p3(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v4p3:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: ds_read_b64 v[2:3], v2 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v4p3:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v4p3:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v4p3:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v4p3:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b128 v[0:3], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <4 x ptr addrspace(3)>, ptr addrspace(2) %ptr, align 16
+ ret <4 x ptr addrspace(3)> %val
+}
+
+define void @store_p1(ptr addrspace(2) %ptr, ptr addrspace(1) %val) {
+; GFX6-LABEL: store_p1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_p1:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_p1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_p1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_p1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b64 v0, v[1:2] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store ptr addrspace(1) %val, ptr addrspace(2) %ptr, align 8
+ ret void
+}
+
+define void @store_p2(ptr addrspace(2) %ptr, ptr addrspace(2) %val) {
+; GFX6-LABEL: store_p2:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_p2:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b32 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_p2:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b32 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_p2:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b32 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_p2:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b32 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store ptr addrspace(2) %val, ptr addrspace(2) %ptr, align 4
+ ret void
+}
+
+define void @store_p3(ptr addrspace(2) %ptr, ptr addrspace(3) %val) {
+; GFX6-LABEL: store_p3:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_p3:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b32 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_p3:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b32 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_p3:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b32 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_p3:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b32 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store ptr addrspace(3) %val, ptr addrspace(2) %ptr, align 4
+ ret void
+}
+
+define void @store_v2p1(ptr addrspace(2) %ptr, <2 x ptr addrspace(1)> %val) {
+; GFX6-LABEL: store_v2p1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 8, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v5, v[3:4] gds
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v2p1:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v2p1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v2p1:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v2p1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b128 v0, v[1:4] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <2 x ptr addrspace(1)> %val, ptr addrspace(2) %ptr, align 16
+ ret void
+}
+
+define void @store_v2p2(ptr addrspace(2) %ptr, <2 x ptr addrspace(2)> %val) {
+; GFX6-LABEL: store_v2p2:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v2p2:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v2p2:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v2p2:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v2p2:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b64 v0, v[1:2] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <2 x ptr addrspace(2)> %val, ptr addrspace(2) %ptr, align 8
+ ret void
+}
+
+define void @store_v2p3(ptr addrspace(2) %ptr, <2 x ptr addrspace(3)> %val) {
+; GFX6-LABEL: store_v2p3:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v2p3:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v2p3:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v2p3:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v2p3:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b64 v0, v[1:2] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <2 x ptr addrspace(3)> %val, ptr addrspace(2) %ptr, align 8
+ ret void
+}
+
+define void @store_v3p3(ptr addrspace(2) %ptr, <3 x ptr addrspace(3)> %val) {
+; GFX6-LABEL: store_v3p3:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 8, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v4, v3 gds
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v3p3:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b96 v0, v[1:3] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v3p3:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b96 v0, v[1:3] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v3p3:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b96 v0, v[1:3] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v3p3:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b96 v0, v[1:3] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <3 x ptr addrspace(3)> %val, ptr addrspace(2) %ptr, align 16
+ ret void
+}
+
+define void @store_v4p3(ptr addrspace(2) %ptr, <4 x ptr addrspace(3)> %val) {
+; GFX6-LABEL: store_v4p3:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 8, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v5, v[3:4] gds
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v4p3:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v4p3:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v4p3:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v4p3:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b128 v0, v[1:4] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <4 x ptr addrspace(3)> %val, ptr addrspace(2) %ptr, align 16
+ ret void
+}
+
+define bfloat @load_bf16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_bf16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_u16 v0, v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_bf16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_u16 v0, v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_u16 v0, v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_u16 v0, v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_u16 v0, v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load bfloat, ptr addrspace(2) %ptr
+ ret bfloat %val
+}
+
+define <2 x half> @load_v2f16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v2f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v2f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b32 v0, v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v2f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b32 v0, v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v2f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b32 v0, v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v2f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b32 v0, v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <2 x half>, ptr addrspace(2) %ptr
+ ret <2 x half> %val
+}
+
+define <3 x half> @load_v3f16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v3f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v3f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v3f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v3f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v3f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b64 v[0:1], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <3 x half>, ptr addrspace(2) %ptr
+ ret <3 x half> %val
+}
+
+define <4 x half> @load_v4f16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v4f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v4f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v4f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v4f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v4f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b64 v[0:1], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <4 x half>, ptr addrspace(2) %ptr
+ ret <4 x half> %val
+}
+
+define <8 x half> @load_v8f16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v8f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: ds_read_b64 v[2:3], v2 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v8f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v8f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v8f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v8f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b128 v[0:3], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <8 x half>, ptr addrspace(2) %ptr
+ ret <8 x half> %val
+}
+
+define <2 x bfloat> @load_v2bf16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v2bf16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v2bf16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b32 v0, v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v2bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b32 v0, v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v2bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b32 v0, v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v2bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b32 v0, v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <2 x bfloat>, ptr addrspace(2) %ptr
+ ret <2 x bfloat> %val
+}
+
+define <3 x bfloat> @load_v3bf16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v3bf16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v3bf16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v3bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v3bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v3bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b64 v[0:1], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <3 x bfloat>, ptr addrspace(2) %ptr
+ ret <3 x bfloat> %val
+}
+
+define <4 x bfloat> @load_v4bf16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v4bf16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v4bf16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v4bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v4bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v4bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b64 v[0:1], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <4 x bfloat>, ptr addrspace(2) %ptr
+ ret <4 x bfloat> %val
+}
+
+define <8 x bfloat> @load_v8bf16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: load_v8bf16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: ds_read_b64 v[2:3], v2 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: load_v8bf16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_v8bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: load_v8bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_read_b128 v[0:3], v0 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: load_v8bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_load_b128 v[0:3], v0 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load <8 x bfloat>, ptr addrspace(2) %ptr
+ ret <8 x bfloat> %val
+}
+
+define void @store_bf16(ptr addrspace(2) %ptr, bfloat %val) {
+; GFX6-LABEL: store_bf16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b16 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_bf16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b16 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b16 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b16 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b16 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store bfloat %val, ptr addrspace(2) %ptr
+ ret void
+}
+
+define void @store_v2f16(ptr addrspace(2) %ptr, <2 x half> %val) {
+; GFX6-LABEL: store_v2f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v2f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b32 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v2f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b32 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v2f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b32 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v2f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b32 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <2 x half> %val, ptr addrspace(2) %ptr
+ ret void
+}
+
+define void @store_v3f16(ptr addrspace(2) %ptr, <3 x half> %val) {
+; GFX6-LABEL: store_v3f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 4, v0
+; GFX6-NEXT: ds_write_b16 v0, v2 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v3f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b16 v0, v2 offset:4 gds
+; GFX7-NEXT: ds_write_b32 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v3f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b16 v0, v2 offset:4 gds
+; GFX9-NEXT: ds_write_b32 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v3f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b16 v0, v2 offset:4 gds
+; GFX10-NEXT: ds_write_b32 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v3f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b16 v0, v2 offset:4 gds
+; GFX11-NEXT: ds_store_b32 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <3 x half> %val, ptr addrspace(2) %ptr
+ ret void
+}
+
+define void @store_v4f16(ptr addrspace(2) %ptr, <4 x half> %val) {
+; GFX6-LABEL: store_v4f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v4f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v4f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v4f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v4f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b64 v0, v[1:2] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <4 x half> %val, ptr addrspace(2) %ptr
+ ret void
+}
+
+define void @store_v8f16(ptr addrspace(2) %ptr, <8 x half> %val) {
+; GFX6-LABEL: store_v8f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 8, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v5, v[3:4] gds
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v8f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v8f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v8f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v8f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b128 v0, v[1:4] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <8 x half> %val, ptr addrspace(2) %ptr
+ ret void
+}
+
+define void @store_v2bf16(ptr addrspace(2) %ptr, <2 x bfloat> %val) {
+; GFX6-LABEL: store_v2bf16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v2bf16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b32 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v2bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b32 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v2bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b32 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v2bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b32 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <2 x bfloat> %val, ptr addrspace(2) %ptr
+ ret void
+}
+
+define void @store_v3bf16(ptr addrspace(2) %ptr, <3 x bfloat> %val) {
+; GFX6-LABEL: store_v3bf16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 4, v0
+; GFX6-NEXT: ds_write_b16 v0, v2 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v3bf16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b16 v0, v2 offset:4 gds
+; GFX7-NEXT: ds_write_b32 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v3bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b16 v0, v2 offset:4 gds
+; GFX9-NEXT: ds_write_b32 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v3bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b16 v0, v2 offset:4 gds
+; GFX10-NEXT: ds_write_b32 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v3bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b16 v0, v2 offset:4 gds
+; GFX11-NEXT: ds_store_b32 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <3 x bfloat> %val, ptr addrspace(2) %ptr
+ ret void
+}
+
+define void @store_v4bf16(ptr addrspace(2) %ptr, <4 x bfloat> %val) {
+; GFX6-LABEL: store_v4bf16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v4bf16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v4bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v4bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v4bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b64 v0, v[1:2] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <4 x bfloat> %val, ptr addrspace(2) %ptr
+ ret void
+}
+
+define void @store_v8bf16(ptr addrspace(2) %ptr, <8 x bfloat> %val) {
+; GFX6-LABEL: store_v8bf16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 8, v0
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v5, v[3:4] gds
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: store_v8bf16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_v8bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: store_v8bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: ds_write_b128 v0, v[1:4] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: store_v8bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: ds_store_b128 v0, v[1:4] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store <8 x bfloat> %val, ptr addrspace(2) %ptr
+ ret void
+}
+
+define half @atomic_load_f16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_u16 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_u16 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_u16 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_u16 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_u16 v0, v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic half, ptr addrspace(2) %ptr seq_cst, align 2
+ ret half %val
+}
+
+define bfloat @atomic_load_bf16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_bf16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_u16 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_bf16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_u16 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_u16 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_u16 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_u16 v0, v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic bfloat, ptr addrspace(2) %ptr seq_cst, align 2
+ ret bfloat %val
+}
+
+define <2 x half> @atomic_load_v2f16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_v2f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b32 v0, v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_v2f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b32 v0, v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_v2f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b32 v0, v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_v2f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b32 v0, v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_v2f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_b32 v0, v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic <2 x half>, ptr addrspace(2) %ptr seq_cst, align 4
+ ret <2 x half> %val
+}
+
+define <4 x half> @atomic_load_v4f16(ptr addrspace(2) %ptr) {
+; GFX6-LABEL: atomic_load_v4f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_load_v4f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1_vol
+; GFX7-NEXT: s_waitcnt expcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_load_v4f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: buffer_wbinvl1_vol
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_load_v4f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_read_b64 v[0:1], v0 gds
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_waitcnt expcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_load_v4f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_load_b64 v[0:1], v0 gds
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: s_waitcnt expcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %val = load atomic <4 x half>, ptr addrspace(2) %ptr seq_cst, align 8
+ ret <4 x half> %val
+}
+
+define void @atomic_store_f16(ptr addrspace(2) %ptr, half %val) {
+; GFX6-LABEL: atomic_store_f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b16 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b16 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b16 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b16 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b16 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic half %val, ptr addrspace(2) %ptr seq_cst, align 2
+ ret void
+}
+
+define void @atomic_store_bf16(ptr addrspace(2) %ptr, bfloat %val) {
+; GFX6-LABEL: atomic_store_bf16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b16 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_bf16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b16 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b16 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b16 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b16 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic bfloat %val, ptr addrspace(2) %ptr seq_cst, align 2
+ ret void
+}
+
+define void @atomic_store_v2f16(ptr addrspace(2) %ptr, <2 x half> %val) {
+; GFX6-LABEL: atomic_store_v2f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b32 v0, v1 gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_v2f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b32 v0, v1 gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_v2f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b32 v0, v1 gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_v2f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b32 v0, v1 gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_v2f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b32 v0, v1 gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic <2 x half> %val, ptr addrspace(2) %ptr seq_cst, align 4
+ ret void
+}
+
+define void @atomic_store_v4f16(ptr addrspace(2) %ptr, <4 x half> %val) {
+; GFX6-LABEL: atomic_store_v4f16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, 0
+; GFX6-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX6-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: atomic_store_v4f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 m0, 0
+; GFX7-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX7-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: atomic_store_v4f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 m0, 0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX9-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: atomic_store_v4f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 m0, 0
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: ds_write_b64 v0, v[1:2] gds
+; GFX10-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: atomic_store_v4f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 m0, 0
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: ds_store_b64 v0, v[1:2] gds
+; GFX11-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ store atomic <4 x half> %val, ptr addrspace(2) %ptr seq_cst, align 8
+ ret void
+}
>From 6d7ccbf5fe08483c5629057beaf42a4e589f237a Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron at aaronballman.com>
Date: Tue, 6 Oct 2026 15:51:35 -0400
Subject: [PATCH 08/46] [clang] Fix handling of invalid nonnull attribute
arguments (#229519)
The nonnull attribute accepts a variadic list of indexes to function
pointer parameters, specifying which ones are not allowed to take a null
value. If an index is invalid (e.g. the referenced function parameter is
not a pointer), then the index is dropped from the semantic attribute
retained in the AST.
Previously, if all the specified indicies were invalid, Clang would
associate a semantic nonnull attribute with no arguments on the
FunctionDecl, which would cause all pointer arguments to be treated as
nonnull. Now, if there are no valid indicies specified, the attribute is
dropped entirely and so no pointer arguments are treated as nonnull.
Fixes #228670
---
clang/docs/ReleaseNotes.md | 4 ++++
clang/lib/Sema/SemaDeclAttr.cpp | 10 ++++++++++
clang/test/Sema/nonnull.c | 20 +++++++++++++++++++-
3 files changed, 33 insertions(+), 1 deletion(-)
diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md
index f2d8d83670b681..c3ed90f2492aad 100644
--- a/clang/docs/ReleaseNotes.md
+++ b/clang/docs/ReleaseNotes.md
@@ -315,6 +315,10 @@ features cannot lower the translation-unit ABI level;
- Clang now accepts `_single_inheritance` under `-fms-compatibility` as an alias for `__single_inheritance`; `_multiple_inheritance` and `_virtual_inheritance` were already correctly supported as aliases.
+- Fixed a bug with handling a `nonnull` attribute with an invalid argument
+ index such that it would inadvertantly apply the attribute with no arguments,
+ causing all function parameters of pointer type to be considered nonnull. (#GH228670)
+
### Improvements to Clang's diagnostics
- `-Wfortify-source` now diagnoses when `strlcat`, `__builtin_strlcat`, `strlcpy`, or
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index eb4a8c2ab9ae01..185a316bcc77c9 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -1377,6 +1377,16 @@ static void handleNonNullAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
NonNullArgs.push_back(Idx);
}
+ // If an argument was specified and there was an attribute ignored warning
+ // issued for it, do not apply the nonnull attribute without any arguments as
+ // that has incorrect semantics in a function like:
+ // __attribute__((nonnull(1))) void f(int val, int *ptr);
+ // because that will signal that 'ptr' is nonnull when it's not intended to
+ // be marked as such. However, continue on if there is at least one valid
+ // parameter index.
+ if (AL.getNumArgs() != 0 && NonNullArgs.empty())
+ return;
+
// If no arguments were specified to __attribute__((nonnull)) then all pointer
// arguments have a nonnull attribute; warn if there aren't any. Skip this
// check if the attribute came from a macro expansion or a template
diff --git a/clang/test/Sema/nonnull.c b/clang/test/Sema/nonnull.c
index 9f3ce2623480fa..9098bf0ecb915c 100644
--- a/clang/test/Sema/nonnull.c
+++ b/clang/test/Sema/nonnull.c
@@ -23,7 +23,7 @@ int main(void) {
}
void foo(const char *str) __attribute__((nonnull("foo"))); // expected-error{{'nonnull' attribute requires parameter 1 to be an integer constant}}
-void bar(int i) __attribute__((nonnull(1))); // expected-warning {{'nonnull' attribute only applies to pointer arguments}} expected-warning {{'nonnull' attribute applied to function with no pointer arguments}}
+void bar(int i) __attribute__((nonnull(1))); // expected-warning {{'nonnull' attribute only applies to pointer arguments}}
void baz(__attribute__((nonnull)) const char *str);
void baz2(__attribute__((nonnull(1))) const char *str); // expected-warning {{'nonnull' attribute when used on parameters takes no arguments}}
@@ -181,3 +181,21 @@ void gh176638_1(int (*g)(const char *h, ...) __attribute__((nonnull(2147483648))
void gh176638_2(int (*g)(const char *h, ...) __attribute__((nonnull(1073741825))) __attribute__((nonnull))) {} // expected-error {{attribute parameter 1 is out of bounds}}
void gh176638_3(int (*g)(const char *h, ...) __attribute__((nonnull(1073741824))) __attribute__((nonnull))) {} // expected-error {{attribute parameter 1 is out of bounds}}
void gh176638_4(int (*g)(const char *h, ...) __attribute__((nonnull(1073741823))) __attribute__((nonnull))) {} // no-warning
+
+__attribute__((nonnull(1))) int gh228670_1(int p0, char *p1) { // expected-warning {{'nonnull' attribute only applies to pointer arguments}}
+ // Previously, the ignored attribute was ending up being applied to the
+ // function without any arguments, meaning all parameters are considered
+ // nonnull. That would in turn generate a warning diagnostic about p1 always
+ // being nonnull. This demonstrates that the attribute is being correctly
+ // ignored.
+ return p1 ? 1 : 0;
+}
+
+__attribute__((nonnull(1, 3))) int gh228670_2(int p0, char *p1, char *p2) { // expected-warning {{'nonnull' attribute only applies to pointer arguments}} \
+ expected-note {{declared 'nonnull' here}}
+ // Similar to the previous test, this verifies that the attribute is not
+ // applied to every parameter, but that the third parameter is still handled
+ // correctly as being non-null because it is explicitly specified.
+ (void)(p1 ? 1 : 0); // No warning here because p1 isn't marked
+ (void)(p2 ? 1 : 0); // expected-warning {{nonnull parameter 'p2' will evaluate to 'true' on first encounter}}
+}
>From 80160224e762d6617f85c0db49b9a7a26a23ffd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miko=C5=82aj=20Pir=C3=B3g?= <mikolaj.maciej.pirog at intel.com>
Date: Tue, 6 Oct 2026 21:52:39 +0200
Subject: [PATCH 09/46] [NewPM] Properly initialize NewPM reg-bank-select
(#229534)
The NewPM initialization of the pass didn't call computeOptMode as the
legacy one does. Missed in porting:
https://github.com/llvm/llvm-project/pull/217756
---
llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
index 261193cd790abc..bac2f52c9ede0d 100644
--- a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
@@ -1774,7 +1774,7 @@ bool RegBankSelectLegacy::runOnMachineFunction(MachineFunction &MF) {
}
RegBankSelectPass::RegBankSelectPass(RegBankSelectMode RunningMode)
- : OptMode(RunningMode) {}
+ : OptMode(computeOptMode(RunningMode)) {}
PreservedAnalyses RegBankSelectPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
>From afae6a07030f30b5dc834fef7a6df709ec88a559 Mon Sep 17 00:00:00 2001
From: Alexander Shaposhnikov <ashaposhnikov at google.com>
Date: Tue, 6 Oct 2026 12:54:37 -0700
Subject: [PATCH 10/46] [SPIRV] Splat scalar condition for vector OpSelect
prior to SPIR-V 1.4 (#229267)
Before SPIR-V 1.4, the Condition operand of an OpSelect instruction with
a vector Result Type must be a vector of Booleans with the same number
of components as the result vector. LLVM IR allows a scalar i1 condition
when selecting between vectors, which causes spirv-val to reject the
module on SPIR-V 1.0 through 1.3 targets.
When targeting SPIR-V versions below 1.4, splat a scalar boolean
condition into a matching boolean vector via OpCompositeConstruct before
emitting OpSelect.
---
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 46 ++++++++++++------
.../extensions/SPV_EXT_long_vector/select.ll | 36 ++++++++++++++
.../SPV_INTEL_masked_gather_scatter/select.ll | 47 +++++++++++++++++++
.../test/CodeGen/SPIRV/instructions/select.ll | 40 ++++++++++++++--
4 files changed, 149 insertions(+), 20 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_long_vector/select.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/select.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 98345a087d8dc4..90b660a792c882 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -496,6 +496,8 @@ class SPIRVInstructionSelector : public InstructionSelector {
Register buildOnesVal(bool AllOnes, SPIRVTypeInst ResType,
MachineInstr &I) const;
Register buildOnesValF(SPIRVTypeInst ResType, MachineInstr &I) const;
+ Register buildVectorSplat(Register ScalarReg, unsigned NumElts,
+ MachineInstr &I) const;
bool wrapIntoSpecConstantOp(MachineInstr &I,
SmallVector<Register> &CompositeArgs) const;
@@ -1682,18 +1684,7 @@ bool SPIRVInstructionSelector::selectLdexp(Register ResVReg,
if (ResType->getOpcode() == SPIRV::OpTypeVector &&
ExpType->getOpcode() != SPIRV::OpTypeVector) {
unsigned NumElts = ResType->getOperand(2).getImm();
- SPIRVTypeInst ExpVecType =
- GR.getOrCreateSPIRVVectorType(ExpType, NumElts, I, TII);
- Register SplatReg = MRI->createVirtualRegister(GR.getRegClass(ExpVecType));
- GR.assignSPIRVTypeToVReg(ExpVecType, SplatReg, MRI->getMF());
- auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
- TII.get(SPIRV::OpCompositeConstruct))
- .addDef(SplatReg)
- .addUse(GR.getSPIRVTypeID(ExpVecType));
- for (unsigned J = 0; J < NumElts; ++J)
- MIB.addUse(ExpReg);
- MIB.constrainAllUses(TII, TRI, RBI);
- ExpReg = SplatReg;
+ ExpReg = buildVectorSplat(ExpReg, NumElts, I);
}
return selectExtInst(ResVReg, ResType, I, CL::ldexp, GL::Ldexp,
@@ -4724,6 +4715,23 @@ Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
return GR.getOrCreateConstInt(One, I, ResType, TII);
}
+Register SPIRVInstructionSelector::buildVectorSplat(Register ScalarReg,
+ unsigned NumElts,
+ MachineInstr &I) const {
+ SPIRVTypeInst VecType = GR.getOrCreateSPIRVVectorType(
+ GR.getSPIRVTypeForVReg(ScalarReg), NumElts, I, TII);
+ Register SplatReg = MRI->createVirtualRegister(GR.getRegClass(VecType));
+ GR.assignSPIRVTypeToVReg(VecType, SplatReg, MRI->getMF());
+ auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
+ TII.get(SPIRV::OpCompositeConstruct))
+ .addDef(SplatReg)
+ .addUse(GR.getSPIRVTypeID(VecType));
+ for (unsigned J = 0; J < NumElts; ++J)
+ MIB.addUse(ScalarReg);
+ MIB.constrainAllUses(TII, TRI, RBI);
+ return SplatReg;
+}
+
bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
SPIRVTypeInst ResType,
MachineInstr &I) const {
@@ -4737,10 +4745,18 @@ bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
bool IsPtrTy =
GR.isScalarOrVectorOfType(SelectFirstArg, SPIRV::OpTypePointer);
- bool IsScalarBool =
- GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool);
+ Register CondReg = I.getOperand(1).getReg();
+ bool IsScalarBool = GR.isScalarOfType(CondReg, SPIRV::OpTypeBool);
unsigned Opcode;
if (isVectorType(GR.getSPIRVTypeForVReg(SelectFirstArg))) {
+ // Before SPIR-V 1.4, the condition of an OpSelect with a vector result
+ // must be a vector of Booleans with the same number of components, while
+ // LLVM IR also allows a scalar i1 condition. Splat it in that case.
+ if (IsScalarBool && !STI.isAtLeastSPIRVVer(VersionTuple(1, 4))) {
+ unsigned NumElts = GR.getScalarOrVectorComponentCount(ResType);
+ CondReg = buildVectorSplat(CondReg, NumElts, I);
+ IsScalarBool = false;
+ }
if (IsFloatTy) {
Opcode = IsScalarBool ? SPIRV::OpSelectVFSCond : SPIRV::OpSelectVFVCond;
} else if (IsPtrTy) {
@@ -4762,7 +4778,7 @@ bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
.addDef(ResVReg)
.addUse(GR.getSPIRVTypeID(ResType))
- .addUse(I.getOperand(1).getReg())
+ .addUse(CondReg)
.addUse(SelectFirstArg)
.addUse(SelectSecondArg)
.constrainAllUses(TII, TRI, RBI);
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_long_vector/select.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_long_vector/select.ll
new file mode 100644
index 00000000000000..7bb152b0761604
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_long_vector/select.ll
@@ -0,0 +1,36 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32v1.3-unknown-unknown --spirv-ext=+SPV_EXT_long_vector %s -o - | FileCheck %s --check-prefixes=CHECK,SPV13
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32v1.3-unknown-unknown --spirv-ext=+SPV_EXT_long_vector %s -o - -filetype=obj | spirv-val --target-env spv1.3 %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64v1.3-unknown-unknown --spirv-ext=+SPV_EXT_long_vector %s -o - | FileCheck %s --check-prefixes=CHECK,SPV13
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64v1.3-unknown-unknown --spirv-ext=+SPV_EXT_long_vector %s -o - -filetype=obj | spirv-val --target-env spv1.3 %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32v1.4-unknown-unknown --spirv-ext=+SPV_EXT_long_vector %s -o - | FileCheck %s --check-prefixes=CHECK,SPV14
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32v1.4-unknown-unknown --spirv-ext=+SPV_EXT_long_vector %s -o - -filetype=obj | spirv-val --target-env spv1.4 %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64v1.4-unknown-unknown --spirv-ext=+SPV_EXT_long_vector %s -o - | FileCheck %s --check-prefixes=CHECK,SPV14
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64v1.4-unknown-unknown --spirv-ext=+SPV_EXT_long_vector %s -o - -filetype=obj | spirv-val --target-env spv1.4 %}
+
+; The component count of OpTypeVectorIdEXT is an ID, not a literal.
+; SPV_EXT_long_vector requires SPIR-V 1.3. In that version, splat the scalar
+; condition to a matching boolean vector; SPIR-V 1.4 allows a scalar condition.
+; CHECK: OpCapability LongVectorEXT
+; CHECK: OpExtension "SPV_EXT_long_vector"
+; CHECK-DAG: [[BOOL:%.+]] = OpTypeBool
+; CHECK-DAG: [[INT:%.+]] = OpTypeInt 32 0
+; CHECK-DAG: [[FIVE:%.+]] = OpConstant [[INT]] 5
+; CHECK-DAG: [[V5INT:%.+]] = OpTypeVectorIdEXT [[INT]] [[FIVE]]
+; SPV13-DAG: [[V5BOOL:%.+]] = OpTypeVectorIdEXT [[BOOL]] [[FIVE]]
+
+; CHECK: OpFunction [[V5INT]]
+; CHECK-NEXT: [[C:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK-NEXT: [[T:%.+]] = OpFunctionParameter [[V5INT]]
+; CHECK-NEXT: [[F:%.+]] = OpFunctionParameter [[V5INT]]
+; CHECK: OpLabel
+; SPV13: [[SPLAT:%.+]] = OpCompositeConstruct [[V5BOOL]] [[C]] [[C]] [[C]] [[C]] [[C]]
+; SPV13-NEXT: [[R:%.+]] = OpSelect [[V5INT]] [[SPLAT]] [[T]] [[F]]
+; SPV14-NOT: OpCompositeConstruct
+; SPV14: [[R:%.+]] = OpSelect [[V5INT]] [[C]] [[T]] [[F]]
+; CHECK-NEXT: OpReturnValue [[R]]
+; CHECK-NEXT: OpFunctionEnd
+define <5 x i32> @select_i32v5(i1 %c, <5 x i32> %t, <5 x i32> %f) {
+ %r = select i1 %c, <5 x i32> %t, <5 x i32> %f
+ ret <5 x i32> %r
+}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/select.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/select.ll
new file mode 100644
index 00000000000000..a63386bfc64578
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_masked_gather_scatter/select.ll
@@ -0,0 +1,47 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32v1.3-unknown-unknown --spirv-ext=+SPV_INTEL_masked_gather_scatter %s -o - | FileCheck %s --check-prefixes=CHECK,SPV13
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32v1.3-unknown-unknown --spirv-ext=+SPV_INTEL_masked_gather_scatter %s -o - -filetype=obj | spirv-val --target-env spv1.3 %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64v1.3-unknown-unknown --spirv-ext=+SPV_INTEL_masked_gather_scatter %s -o - | FileCheck %s --check-prefixes=CHECK,SPV13
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64v1.3-unknown-unknown --spirv-ext=+SPV_INTEL_masked_gather_scatter %s -o - -filetype=obj | spirv-val --target-env spv1.3 %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32v1.4-unknown-unknown --spirv-ext=+SPV_INTEL_masked_gather_scatter %s -o - | FileCheck %s --check-prefixes=CHECK,SPV14
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32v1.4-unknown-unknown --spirv-ext=+SPV_INTEL_masked_gather_scatter %s -o - -filetype=obj | spirv-val --target-env spv1.4 %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64v1.4-unknown-unknown --spirv-ext=+SPV_INTEL_masked_gather_scatter %s -o - | FileCheck %s --check-prefixes=CHECK,SPV14
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64v1.4-unknown-unknown --spirv-ext=+SPV_INTEL_masked_gather_scatter %s -o - -filetype=obj | spirv-val --target-env spv1.4 %}
+
+; Vectors of pointers require SPV_INTEL_masked_gather_scatter.
+; CHECK: OpCapability MaskedGatherScatterINTEL
+; CHECK: OpExtension "SPV_INTEL_masked_gather_scatter"
+; CHECK-DAG: [[BOOL:%.+]] = OpTypeBool
+; CHECK-DAG: [[V2BOOL:%.+]] = OpTypeVector [[BOOL]] 2
+; CHECK-DAG: [[PTR:%.+]] = OpTypePointer CrossWorkgroup {{%.+}}
+; CHECK-DAG: [[V2PTR:%.+]] = OpTypeVector [[PTR]] 2
+
+; CHECK: OpFunction [[V2PTR]]
+; CHECK-NEXT: [[C:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK-NEXT: [[T:%.+]] = OpFunctionParameter [[V2PTR]]
+; CHECK-NEXT: [[F:%.+]] = OpFunctionParameter [[V2PTR]]
+; CHECK: OpLabel
+; SPV13: [[SPLAT:%.+]] = OpCompositeConstruct [[V2BOOL]] [[C]] [[C]]
+; SPV13-NEXT: [[R:%.+]] = OpSelect [[V2PTR]] [[SPLAT]] [[T]] [[F]]
+; SPV14-NOT: OpCompositeConstruct
+; SPV14: [[R:%.+]] = OpSelect [[V2PTR]] [[C]] [[T]] [[F]]
+; CHECK-NEXT: OpReturnValue [[R]]
+; CHECK-NEXT: OpFunctionEnd
+define <2 x ptr addrspace(1)> @select_ptrv2(i1 %c, <2 x ptr addrspace(1)> %t, <2 x ptr addrspace(1)> %f) {
+ %r = select i1 %c, <2 x ptr addrspace(1)> %t, <2 x ptr addrspace(1)> %f
+ ret <2 x ptr addrspace(1)> %r
+}
+
+; CHECK: OpFunction [[V2PTR]]
+; CHECK-NEXT: [[C:%.+]] = OpFunctionParameter [[V2BOOL]]
+; CHECK-NEXT: [[T:%.+]] = OpFunctionParameter [[V2PTR]]
+; CHECK-NEXT: [[F:%.+]] = OpFunctionParameter [[V2PTR]]
+; CHECK: OpLabel
+; CHECK-NOT: OpCompositeConstruct
+; CHECK: [[R:%.+]] = OpSelect [[V2PTR]] [[C]] [[T]] [[F]]
+; CHECK-NEXT: OpReturnValue [[R]]
+; CHECK-NEXT: OpFunctionEnd
+define <2 x ptr addrspace(1)> @select_v2ptrv2(<2 x i1> %c, <2 x ptr addrspace(1)> %t, <2 x ptr addrspace(1)> %f) {
+ %r = select <2 x i1> %c, <2 x ptr addrspace(1)> %t, <2 x ptr addrspace(1)> %f
+ ret <2 x ptr addrspace(1)> %r
+}
diff --git a/llvm/test/CodeGen/SPIRV/instructions/select.ll b/llvm/test/CodeGen/SPIRV/instructions/select.ll
index 91d5f12c4b7ba7..08f0e3ba2841ae 100644
--- a/llvm/test/CodeGen/SPIRV/instructions/select.ll
+++ b/llvm/test/CodeGen/SPIRV/instructions/select.ll
@@ -1,14 +1,24 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32v1.4-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,SPV14
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32v1.4-unknown-unknown %s -o - -filetype=obj | spirv-val --target-env spv1.4 %}
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64v1.4-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,SPV14
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64v1.4-unknown-unknown %s -o - -filetype=obj | spirv-val --target-env spv1.4 %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32v1.3-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,SPV13
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32v1.3-unknown-unknown %s -o - -filetype=obj | spirv-val --target-env spv1.3 %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64v1.3-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,SPV13
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64v1.3-unknown-unknown %s -o - -filetype=obj | spirv-val --target-env spv1.3 %}
; CHECK-DAG: OpName [[SCALARi32:%.+]] "select_i32"
; CHECK-DAG: OpName [[SCALARPTR:%.+]] "select_ptr"
; CHECK-DAG: OpName [[VEC2i32:%.+]] "select_i32v2"
+; CHECK-DAG: OpName [[VEC2f32:%.+]] "select_f32v2"
; CHECK-DAG: OpName [[VEC2i32v2:%.+]] "select_v2i32v2"
+; CHECK-DAG: [[BOOL:%.+]] = OpTypeBool
+; CHECK-DAG: [[V2BOOL:%.+]] = OpTypeVector [[BOOL]] 2
+
; CHECK: [[SCALARi32]] = OpFunction
; CHECK-NEXT: [[C:%.+]] = OpFunctionParameter
; CHECK-NEXT: [[T:%.+]] = OpFunctionParameter
@@ -40,7 +50,10 @@ define ptr @select_ptr(i1 %c, ptr %t, ptr %f) {
; CHECK-NEXT: [[T:%.+]] = OpFunctionParameter
; CHECK-NEXT: [[F:%.+]] = OpFunctionParameter
; CHECK: OpLabel
-; CHECK: [[R:%.+]] = OpSelect {{%.+}} [[C]] [[T]] [[F]]
+; SPV13: [[CSPLAT:%.+]] = OpCompositeConstruct [[V2BOOL]] [[C]] [[C]]
+; SPV13: [[R:%.+]] = OpSelect {{%.+}} [[CSPLAT]] [[T]] [[F]]
+; SPV14-NOT: OpCompositeConstruct
+; SPV14: [[R:%.+]] = OpSelect {{%.+}} [[C]] [[T]] [[F]]
; CHECK: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define <2 x i32> @select_i32v2(i1 %c, <2 x i32> %t, <2 x i32> %f) {
@@ -48,11 +61,28 @@ define <2 x i32> @select_i32v2(i1 %c, <2 x i32> %t, <2 x i32> %f) {
ret <2 x i32> %r
}
+; CHECK: [[VEC2f32]] = OpFunction
+; CHECK-NEXT: [[C:%.+]] = OpFunctionParameter
+; CHECK-NEXT: [[T:%.+]] = OpFunctionParameter
+; CHECK-NEXT: [[F:%.+]] = OpFunctionParameter
+; CHECK: OpLabel
+; SPV13: [[CSPLAT:%.+]] = OpCompositeConstruct [[V2BOOL]] [[C]] [[C]]
+; SPV13: [[R:%.+]] = OpSelect {{%.+}} [[CSPLAT]] [[T]] [[F]]
+; SPV14-NOT: OpCompositeConstruct
+; SPV14: [[R:%.+]] = OpSelect {{%.+}} [[C]] [[T]] [[F]]
+; CHECK: OpReturnValue [[R]]
+; CHECK-NEXT: OpFunctionEnd
+define <2 x float> @select_f32v2(i1 %c, <2 x float> %t, <2 x float> %f) {
+ %r = select i1 %c, <2 x float> %t, <2 x float> %f
+ ret <2 x float> %r
+}
+
; CHECK: [[VEC2i32v2]] = OpFunction
; CHECK-NEXT: [[C:%.+]] = OpFunctionParameter
; CHECK-NEXT: [[T:%.+]] = OpFunctionParameter
; CHECK-NEXT: [[F:%.+]] = OpFunctionParameter
; CHECK: OpLabel
+; CHECK-NOT: OpCompositeConstruct
; CHECK: [[R:%.+]] = OpSelect {{%.+}} [[C]] [[T]] [[F]]
; CHECK: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
>From 26dedddd9f40b3eb344a8f47478e6320b595257f Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas at devlieghere.com>
Date: Tue, 6 Oct 2026 12:55:38 -0700
Subject: [PATCH 11/46] [lldb] Add StringPool and per-debugger and system pool
handles (NFC) (#228611)
Move the ConstString pool into a StringPool class and add a non-owning
StringPoolRef handle. This is in preparation for having one global and
multiple per-Debugger ConstString pool. Strings at the SB API layer will
be interned into the appropriate pool. As long as these remain
references, this operation is a NOOP.
---
lldb/include/lldb/Core/Debugger.h | 6 +
lldb/include/lldb/Utility/StringPool.h | 86 ++++++
.../SystemInitializerCommon.cpp | 4 +
lldb/source/Utility/CMakeLists.txt | 1 +
lldb/source/Utility/ConstString.cpp | 246 ++---------------
lldb/source/Utility/StringPool.cpp | 252 ++++++++++++++++++
lldb/unittests/Utility/CMakeLists.txt | 1 +
lldb/unittests/Utility/StringPoolTest.cpp | 74 +++++
8 files changed, 439 insertions(+), 231 deletions(-)
create mode 100644 lldb/include/lldb/Utility/StringPool.h
create mode 100644 lldb/source/Utility/StringPool.cpp
create mode 100644 lldb/unittests/Utility/StringPoolTest.cpp
diff --git a/lldb/include/lldb/Core/Debugger.h b/lldb/include/lldb/Core/Debugger.h
index d3352504497e43..bcc14479c3360d 100644
--- a/lldb/include/lldb/Core/Debugger.h
+++ b/lldb/include/lldb/Core/Debugger.h
@@ -34,6 +34,7 @@
#include "lldb/Utility/ConstString.h"
#include "lldb/Utility/FileSpec.h"
#include "lldb/Utility/Status.h"
+#include "lldb/Utility/StringPool.h"
#include "lldb/Utility/StructuredData.h"
#include "lldb/Utility/UserID.h"
#include "lldb/lldb-defines.h"
@@ -418,6 +419,10 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
const std::string &GetInstanceName() const { return m_instance_name; }
+ /// The pool for strings handed out through the SB API that belong to this
+ /// debugger.
+ StringPoolRef GetStringPool() const { return m_string_pool; }
+
bool GetShowInlineDiagnostics() const;
bool SetShowInlineDiagnostics(bool);
@@ -814,6 +819,7 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
llvm::StringMap<std::weak_ptr<LogHandler>> m_stream_handlers;
std::shared_ptr<CallbackLogHandler> m_callback_handler_sp;
const std::string m_instance_name;
+ StringPoolRef m_string_pool;
static LoadPluginCallbackType g_load_plugin_callback;
typedef std::vector<llvm::sys::DynamicLibrary> LoadedPluginsList;
LoadedPluginsList m_loaded_plugins;
diff --git a/lldb/include/lldb/Utility/StringPool.h b/lldb/include/lldb/Utility/StringPool.h
new file mode 100644
index 00000000000000..d5a7a5597975ed
--- /dev/null
+++ b/lldb/include/lldb/Utility/StringPool.h
@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_UTILITY_STRINGPOOL_H
+#define LLDB_UTILITY_STRINGPOOL_H
+
+#include "lldb/Utility/ConstString.h"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstddef>
+#include <memory>
+
+namespace lldb_private {
+
+class StringPoolRef;
+
+/// A thread-safe pool of interned, null-terminated strings. A string returned
+/// by a pool stays valid for the lifetime of that pool.
+class StringPool {
+public:
+ StringPool();
+ ~StringPool();
+
+ StringPool(const StringPool &) = delete;
+ StringPool &operator=(const StringPool &) = delete;
+
+ /// The pool backing ConstString. It is never destroyed.
+ static StringPool &GetGlobal();
+
+ /// Set up and tear down the system pool, for strings that are not owned by a
+ /// Debugger.
+ static void Initialize();
+ static void Terminate();
+
+ /// The system pool. Only valid between Initialize and Terminate.
+ static StringPoolRef GetSystem();
+
+ /// Returns the pooled copy of \p str, or nullptr if \p str has no data.
+ const char *Intern(llvm::StringRef str);
+
+ const char *GetConstCString(const char *cstr);
+ const char *GetConstCStringWithLength(const char *cstr, size_t cstr_len);
+ const char *GetConstTrimmedCStringWithLength(const char *cstr,
+ size_t cstr_len);
+
+ /// Interns \p demangled and links it with the already interned \p mangled.
+ const char *GetConstCStringAndSetMangledCounterPart(llvm::StringRef demangled,
+ llvm::StringRef mangled);
+ const char *GetMangledCounterpart(llvm::StringRef str);
+
+ /// Length of a string returned by any pool.
+ static size_t GetConstCStringLength(const char *ccstr);
+
+ ConstString::MemoryStats GetMemoryStats() const;
+
+private:
+ struct PoolEntry;
+
+ PoolEntry &selectPool(uint32_t hash);
+ PoolEntry &selectPool(llvm::StringRef str);
+
+ std::unique_ptr<PoolEntry[]> m_string_pools;
+};
+
+/// A handle to a StringPool that does not own it.
+class StringPoolRef {
+public:
+ /// Refers to the global pool.
+ StringPoolRef() : m_pool(&StringPool::GetGlobal()) {}
+ explicit StringPoolRef(StringPool &pool) : m_pool(&pool) {}
+
+ const char *Intern(llvm::StringRef str) const { return m_pool->Intern(str); }
+
+private:
+ StringPool *m_pool;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_UTILITY_STRINGPOOL_H
diff --git a/lldb/source/Initialization/SystemInitializerCommon.cpp b/lldb/source/Initialization/SystemInitializerCommon.cpp
index 97aad944ece933..b2e5a84746a862 100644
--- a/lldb/source/Initialization/SystemInitializerCommon.cpp
+++ b/lldb/source/Initialization/SystemInitializerCommon.cpp
@@ -15,6 +15,7 @@
#include "lldb/Host/Socket.h"
#include "lldb/Target/Statistics.h"
#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/StringPool.h"
#include "lldb/Utility/Timer.h"
#include "lldb/Version/Version.h"
@@ -40,6 +41,7 @@ SystemInitializerCommon::SystemInitializerCommon() = default;
SystemInitializerCommon::~SystemInitializerCommon() = default;
llvm::Error SystemInitializerCommon::Initialize() {
+ StringPool::Initialize();
#if defined(_WIN32)
const char *disable_crash_dialog_var = getenv("LLDB_DISABLE_CRASH_DIALOG");
if (disable_crash_dialog_var &&
@@ -112,4 +114,6 @@ void SystemInitializerCommon::Terminate() {
FileSystem::Terminate();
Diagnostics::Terminate();
LLDBLogChannel::Terminate();
+
+ StringPool::Terminate();
}
diff --git a/lldb/source/Utility/CMakeLists.txt b/lldb/source/Utility/CMakeLists.txt
index 5033edb8773f77..a13aec8327868b 100644
--- a/lldb/source/Utility/CMakeLists.txt
+++ b/lldb/source/Utility/CMakeLists.txt
@@ -68,6 +68,7 @@ add_lldb_library(lldbUtility NO_INTERNAL_DEPENDENCIES
StringExtractor.cpp
StringExtractorGDBRemote.cpp
StringList.cpp
+ StringPool.cpp
StructuredData.cpp
TildeExpressionResolver.cpp
Timer.cpp
diff --git a/lldb/source/Utility/ConstString.cpp b/lldb/source/Utility/ConstString.cpp
index 8e97b7bb382a2b..9890e46d6b0f1e 100644
--- a/lldb/source/Utility/ConstString.cpp
+++ b/lldb/source/Utility/ConstString.cpp
@@ -9,18 +9,12 @@
#include "lldb/Utility/ConstString.h"
#include "lldb/Utility/Stream.h"
+#include "lldb/Utility/StringPool.h"
-#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator.h"
-#include "llvm/Support/Allocator.h"
-#include "llvm/Support/DJB.h"
#include "llvm/Support/FormatProviders.h"
-#include "llvm/Support/Threading.h"
-#include <array>
-#include <mutex>
-#include <shared_mutex>
#include <utility>
#include <cinttypes>
@@ -29,227 +23,15 @@
using namespace lldb_private;
-#if !defined(__APPLE__)
-using PoolMutex = std::shared_mutex;
-#else
-#include <os/lock.h>
-
-namespace {
-/// On Apple platforms os_unfair_lock is significantly faster than
-/// pthread_rwlock for concurrent writes, and roughly on par for concurrent
-/// reads.
-///
-/// The class satisfies both Lockable and SharedLockable so it composes with
-/// std::lock_guard and std::shared_lock.
-class PoolMutex {
-public:
- void lock() { os_unfair_lock_lock(&m_lock); }
- void unlock() { os_unfair_lock_unlock(&m_lock); }
- void lock_shared() { os_unfair_lock_lock(&m_lock); }
- void unlock_shared() { os_unfair_lock_unlock(&m_lock); }
-
-private:
- os_unfair_lock m_lock = OS_UNFAIR_LOCK_INIT;
-};
-} // namespace
-#endif
-
-class Pool {
-public:
- /// The default BumpPtrAllocatorImpl slab size.
- static const size_t AllocatorSlabSize = 4096;
- static const size_t SizeThreshold = AllocatorSlabSize;
- /// Every Pool has its own allocator which receives an equal share of
- /// the ConstString allocations. This means that when allocating many
- /// ConstStrings, every allocator sees only its small share of allocations and
- /// assumes LLDB only allocated a small amount of memory so far. In reality
- /// LLDB allocated a total memory that is N times as large as what the
- /// allocator sees (where N is the number of string pools). This causes that
- /// the BumpPtrAllocator continues a long time to allocate memory in small
- /// chunks which only makes sense when allocating a small amount of memory
- /// (which is true from the perspective of a single allocator). On some
- /// systems doing all these small memory allocations causes LLDB to spend
- /// a lot of time in malloc, so we need to force all these allocators to
- /// behave like one allocator in terms of scaling their memory allocations
- /// with increased demand. To do this we set the growth delay for each single
- /// allocator to a rate so that our pool of allocators scales their memory
- /// allocations similar to a single BumpPtrAllocatorImpl.
- ///
- /// Currently we have 256 string pools and the normal growth delay of the
- /// BumpPtrAllocatorImpl is 128 (i.e., the memory allocation size increases
- /// every 128 full chunks), so by changing the delay to 1 we get a
- /// total growth delay in our allocator collection of 256/1 = 256. This is
- /// still only half as fast as a normal allocator but we can't go any faster
- /// without decreasing the number of string pools.
- static const size_t AllocatorGrowthDelay = 1;
- typedef llvm::BumpPtrAllocatorImpl<llvm::MallocAllocator, AllocatorSlabSize,
- SizeThreshold, AllocatorGrowthDelay>
- Allocator;
- typedef const char *StringPoolValueType;
- typedef llvm::StringMap<StringPoolValueType, Allocator> StringPool;
- typedef llvm::StringMapEntry<StringPoolValueType> StringPoolEntryType;
-
- static StringPoolEntryType &
- GetStringMapEntryFromKeyData(const char *keyData) {
- return StringPoolEntryType::GetStringMapEntryFromKeyData(keyData);
- }
-
- static size_t GetConstCStringLength(const char *ccstr) {
- if (ccstr != nullptr) {
- // Since the entry is read only, and we derive the entry entirely from
- // the pointer, we don't need the lock.
- const StringPoolEntryType &entry = GetStringMapEntryFromKeyData(ccstr);
- return entry.getKeyLength();
- }
- return 0;
- }
-
- StringPoolValueType GetMangledCounterpart(llvm::StringRef str) {
- const char *const ccstr = str.data();
- if (ccstr != nullptr) {
- const PoolEntry &pool = selectPool(str);
- std::shared_lock<PoolMutex> lock(pool.m_mutex);
- return GetStringMapEntryFromKeyData(ccstr).getValue();
- }
- return nullptr;
- }
-
- const char *GetConstCString(const char *cstr) {
- if (cstr != nullptr)
- return GetConstCStringWithLength(cstr, strlen(cstr));
- return nullptr;
- }
-
- const char *GetConstCStringWithLength(const char *cstr, size_t cstr_len) {
- if (cstr != nullptr)
- return GetConstCStringWithStringRef(llvm::StringRef(cstr, cstr_len));
- return nullptr;
- }
-
- const char *GetConstCStringWithStringRef(llvm::StringRef string_ref) {
- if (string_ref.data()) {
- const uint32_t string_hash = StringPool::hash(string_ref);
- PoolEntry &pool = selectPool(string_hash);
-
- {
- std::shared_lock<PoolMutex> lock(pool.m_mutex);
- auto it = pool.m_string_map.find(string_ref, string_hash);
- if (it != pool.m_string_map.end())
- return it->getKeyData();
- }
-
- std::lock_guard<PoolMutex> lock(pool.m_mutex);
- pool.used_bytes += string_ref.size();
- StringPoolEntryType &entry =
- *pool.m_string_map
- .insert(std::make_pair(string_ref, nullptr), string_hash)
- .first;
- return entry.getKeyData();
- }
- return nullptr;
- }
-
- const char *GetConstCStringAndSetMangledCounterPart(llvm::StringRef demangled,
- llvm::StringRef mangled) {
- const char *demangled_ccstr = nullptr;
- const char *const mangled_ccstr = mangled.data();
-
- {
- const uint32_t demangled_hash = StringPool::hash(demangled);
- PoolEntry &pool = selectPool(demangled_hash);
- std::lock_guard<PoolMutex> lock(pool.m_mutex);
-
- // Make or update string pool entry with the mangled counterpart
- StringPool &map = pool.m_string_map;
- auto [entry, inserted] =
- map.try_emplace_with_hash(demangled, demangled_hash);
- if (inserted)
- pool.used_bytes += demangled.size();
-
- entry->second = mangled_ccstr;
-
- // Extract the const version of the demangled_cstr
- demangled_ccstr = entry->getKeyData();
- }
-
- {
- // Now assign the demangled const string as the counterpart of the
- // mangled const string...
- PoolEntry &pool = selectPool(mangled);
- std::lock_guard<PoolMutex> lock(pool.m_mutex);
- GetStringMapEntryFromKeyData(mangled_ccstr).setValue(demangled_ccstr);
- }
-
- // Return the constant demangled C string
- return demangled_ccstr;
- }
-
- const char *GetConstTrimmedCStringWithLength(const char *cstr,
- size_t cstr_len) {
- if (cstr != nullptr) {
- const size_t trimmed_len = strnlen(cstr, cstr_len);
- return GetConstCStringWithLength(cstr, trimmed_len);
- }
- return nullptr;
- }
-
- ConstString::MemoryStats GetMemoryStats() const {
- ConstString::MemoryStats stats;
- for (const auto &pool : m_string_pools) {
- std::shared_lock<PoolMutex> lock(pool.m_mutex);
- const Allocator &alloc = pool.m_string_map.getAllocator();
- stats.bytes_total += alloc.getTotalMemory();
- stats.bytes_used += pool.used_bytes;
- }
- return stats;
- }
-
-protected:
- struct PoolEntry {
- mutable PoolMutex m_mutex;
- StringPool m_string_map;
- /// The exact number of bytes used by this pool.
- /// This excludes alignment, padding and redzones.
- std::size_t used_bytes = 0;
- };
-
- std::array<PoolEntry, 256> m_string_pools;
-
- PoolEntry &selectPool(const llvm::StringRef &s) {
- return selectPool(StringPool::hash(s));
- }
-
- PoolEntry &selectPool(uint32_t h) {
- return m_string_pools[((h >> 24) ^ (h >> 16) ^ (h >> 8) ^ h) & 0xff];
- }
-};
-
-// Frameworks and dylibs aren't supposed to have global C++ initializers so we
-// hide the string pool in a static function so that it will get initialized on
-// the first call to this static function.
-//
-// Note, for now we make the string pool a pointer to the pool, because we
-// can't guarantee that some objects won't get destroyed after the global
-// destructor chain is run, and trying to make sure no destructors touch
-// ConstStrings is difficult. So we leak the pool instead.
-static Pool &StringPool() {
- static llvm::once_flag g_pool_initialization_flag;
- static Pool *g_string_pool = nullptr;
-
- llvm::call_once(g_pool_initialization_flag,
- []() { g_string_pool = new Pool(); });
-
- return *g_string_pool;
-}
-
ConstString::ConstString(const char *cstr)
- : m_string(StringPool().GetConstCString(cstr)) {}
+ : m_string(StringPool::GetGlobal().GetConstCString(cstr)) {}
ConstString::ConstString(const char *cstr, size_t cstr_len)
- : m_string(StringPool().GetConstCStringWithLength(cstr, cstr_len)) {}
+ : m_string(
+ StringPool::GetGlobal().GetConstCStringWithLength(cstr, cstr_len)) {}
ConstString::ConstString(llvm::StringRef s)
- : m_string(StringPool().GetConstCStringWithStringRef(s)) {}
+ : m_string(StringPool::GetGlobal().Intern(s)) {}
bool ConstString::operator<(ConstString rhs) const {
if (m_string == rhs.m_string)
@@ -275,7 +57,7 @@ Stream &lldb_private::operator<<(Stream &s, ConstString str) {
}
size_t ConstString::GetLength() const {
- return Pool::GetConstCStringLength(m_string);
+ return StringPool::GetConstCStringLength(m_string);
}
bool ConstString::Equals(ConstString lhs, ConstString rhs,
@@ -339,35 +121,37 @@ void ConstString::DumpDebug(Stream *s) const {
}
void ConstString::SetCString(const char *cstr) {
- m_string = StringPool().GetConstCString(cstr);
+ m_string = StringPool::GetGlobal().GetConstCString(cstr);
}
void ConstString::SetString(llvm::StringRef s) {
- m_string = StringPool().GetConstCStringWithStringRef(s);
+ m_string = StringPool::GetGlobal().Intern(s);
}
void ConstString::SetStringWithMangledCounterpart(llvm::StringRef demangled,
ConstString mangled) {
- m_string = StringPool().GetConstCStringAndSetMangledCounterPart(
+ m_string = StringPool::GetGlobal().GetConstCStringAndSetMangledCounterPart(
demangled, mangled.GetStringRef());
}
bool ConstString::GetMangledCounterpart(ConstString &counterpart) const {
- counterpart.m_string = StringPool().GetMangledCounterpart(GetStringRef());
+ counterpart.m_string =
+ StringPool::GetGlobal().GetMangledCounterpart(GetStringRef());
return (bool)counterpart;
}
void ConstString::SetCStringWithLength(const char *cstr, size_t cstr_len) {
- m_string = StringPool().GetConstCStringWithLength(cstr, cstr_len);
+ m_string = StringPool::GetGlobal().GetConstCStringWithLength(cstr, cstr_len);
}
void ConstString::SetTrimmedCStringWithLength(const char *cstr,
size_t cstr_len) {
- m_string = StringPool().GetConstTrimmedCStringWithLength(cstr, cstr_len);
+ m_string =
+ StringPool::GetGlobal().GetConstTrimmedCStringWithLength(cstr, cstr_len);
}
ConstString::MemoryStats ConstString::GetMemoryStats() {
- return StringPool().GetMemoryStats();
+ return StringPool::GetGlobal().GetMemoryStats();
}
void llvm::format_provider<ConstString>::format(const ConstString &CS,
diff --git a/lldb/source/Utility/StringPool.cpp b/lldb/source/Utility/StringPool.cpp
new file mode 100644
index 00000000000000..84cf7df02d9293
--- /dev/null
+++ b/lldb/source/Utility/StringPool.cpp
@@ -0,0 +1,252 @@
+//===-- StringPool.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Utility/StringPool.h"
+
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Threading.h"
+
+#include <array>
+#include <mutex>
+#include <shared_mutex>
+#include <utility>
+
+#include <cstdint>
+#include <cstring>
+
+using namespace lldb_private;
+
+#if !defined(__APPLE__)
+using PoolMutex = std::shared_mutex;
+#else
+#include <os/lock.h>
+
+namespace {
+/// On Apple platforms os_unfair_lock is significantly faster than
+/// pthread_rwlock for concurrent writes, and roughly on par for concurrent
+/// reads.
+///
+/// The class satisfies both Lockable and SharedLockable so it composes with
+/// std::lock_guard and std::shared_lock.
+class PoolMutex {
+public:
+ void lock() { os_unfair_lock_lock(&m_lock); }
+ void unlock() { os_unfair_lock_unlock(&m_lock); }
+ void lock_shared() { os_unfair_lock_lock(&m_lock); }
+ void unlock_shared() { os_unfair_lock_unlock(&m_lock); }
+
+private:
+ os_unfair_lock m_lock = OS_UNFAIR_LOCK_INIT;
+};
+} // namespace
+#endif
+
+namespace {
+/// The default BumpPtrAllocatorImpl slab size.
+constexpr size_t AllocatorSlabSize = 4096;
+constexpr size_t SizeThreshold = AllocatorSlabSize;
+constexpr size_t NumPools = 256;
+/// Every pool shard has its own allocator which receives an equal share of
+/// the string allocations. This means that when allocating many strings, every
+/// allocator sees only its small share of allocations and assumes LLDB only
+/// allocated a small amount of memory so far. In reality LLDB allocated a total
+/// memory that is N times as large as what the allocator sees (where N is the
+/// number of string pools). This causes that the BumpPtrAllocator continues a
+/// long time to allocate memory in small chunks which only makes sense when
+/// allocating a small amount of memory (which is true from the perspective of a
+/// single allocator). On some systems doing all these small memory allocations
+/// causes LLDB to spend a lot of time in malloc, so we need to force all these
+/// allocators to behave like one allocator in terms of scaling their memory
+/// allocations with increased demand. To do this we set the growth delay for
+/// each single allocator to a rate so that our pool of allocators scales their
+/// memory allocations similar to a single BumpPtrAllocatorImpl.
+///
+/// Currently we have 256 string pools and the normal growth delay of the
+/// BumpPtrAllocatorImpl is 128 (i.e., the memory allocation size increases
+/// every 128 full chunks), so by changing the delay to 1 we get a
+/// total growth delay in our allocator collection of 256/1 = 256. This is
+/// still only half as fast as a normal allocator but we can't go any faster
+/// without decreasing the number of string pools.
+constexpr size_t AllocatorGrowthDelay = 1;
+using Allocator =
+ llvm::BumpPtrAllocatorImpl<llvm::MallocAllocator, AllocatorSlabSize,
+ SizeThreshold, AllocatorGrowthDelay>;
+using StringPoolValueType = const char *;
+using StringMapType = llvm::StringMap<StringPoolValueType, Allocator>;
+using StringPoolEntryType = llvm::StringMapEntry<StringPoolValueType>;
+
+StringPoolEntryType &GetStringMapEntryFromKeyData(const char *keyData) {
+ return StringPoolEntryType::GetStringMapEntryFromKeyData(keyData);
+}
+} // namespace
+
+struct StringPool::PoolEntry {
+ mutable PoolMutex m_mutex;
+ StringMapType m_string_map;
+ /// The exact number of bytes used by this pool.
+ /// This excludes alignment, padding and redzones.
+ std::size_t used_bytes = 0;
+};
+
+StringPool::StringPool() : m_string_pools(new PoolEntry[NumPools]) {}
+
+StringPool::~StringPool() = default;
+
+// Frameworks and dylibs aren't supposed to have global C++ initializers so we
+// hide the string pool in a static function so that it will get initialized on
+// the first call to this static function.
+//
+// Note, for now we make the string pool a pointer to the pool, because we
+// can't guarantee that some objects won't get destroyed after the global
+// destructor chain is run, and trying to make sure no destructors touch
+// ConstStrings is difficult. So we leak the pool instead.
+StringPool &StringPool::GetGlobal() {
+ static llvm::once_flag g_pool_initialization_flag;
+ static StringPool *g_string_pool = nullptr;
+
+ llvm::call_once(g_pool_initialization_flag,
+ []() { g_string_pool = new StringPool(); });
+
+ return *g_string_pool;
+}
+
+static StringPool *g_system_pool = nullptr;
+
+void StringPool::Initialize() {
+ assert(!g_system_pool && "system pool already initialized");
+ g_system_pool = &GetGlobal();
+}
+
+void StringPool::Terminate() { g_system_pool = nullptr; }
+
+StringPoolRef StringPool::GetSystem() {
+ assert(g_system_pool && "system pool not initialized");
+ return StringPoolRef(*g_system_pool);
+}
+
+StringPool::PoolEntry &StringPool::selectPool(uint32_t h) {
+ return m_string_pools[((h >> 24) ^ (h >> 16) ^ (h >> 8) ^ h) & 0xff];
+}
+
+StringPool::PoolEntry &StringPool::selectPool(llvm::StringRef s) {
+ return selectPool(StringMapType::hash(s));
+}
+
+size_t StringPool::GetConstCStringLength(const char *ccstr) {
+ if (ccstr != nullptr) {
+ // Since the entry is read only, and we derive the entry entirely from
+ // the pointer, we don't need the lock.
+ const StringPoolEntryType &entry = GetStringMapEntryFromKeyData(ccstr);
+ return entry.getKeyLength();
+ }
+ return 0;
+}
+
+const char *StringPool::GetMangledCounterpart(llvm::StringRef str) {
+ const char *const ccstr = str.data();
+ if (ccstr != nullptr) {
+ const PoolEntry &pool = selectPool(str);
+ std::shared_lock<PoolMutex> lock(pool.m_mutex);
+ return GetStringMapEntryFromKeyData(ccstr).getValue();
+ }
+ return nullptr;
+}
+
+const char *StringPool::GetConstCString(const char *cstr) {
+ if (cstr != nullptr)
+ return GetConstCStringWithLength(cstr, strlen(cstr));
+ return nullptr;
+}
+
+const char *StringPool::GetConstCStringWithLength(const char *cstr,
+ size_t cstr_len) {
+ if (cstr != nullptr)
+ return Intern(llvm::StringRef(cstr, cstr_len));
+ return nullptr;
+}
+
+const char *StringPool::Intern(llvm::StringRef string_ref) {
+ if (string_ref.data()) {
+ const uint32_t string_hash = StringMapType::hash(string_ref);
+ PoolEntry &pool = selectPool(string_hash);
+
+ {
+ std::shared_lock<PoolMutex> lock(pool.m_mutex);
+ auto it = pool.m_string_map.find(string_ref, string_hash);
+ if (it != pool.m_string_map.end())
+ return it->getKeyData();
+ }
+
+ std::lock_guard<PoolMutex> lock(pool.m_mutex);
+ pool.used_bytes += string_ref.size();
+ StringPoolEntryType &entry =
+ *pool.m_string_map
+ .insert(std::make_pair(string_ref, nullptr), string_hash)
+ .first;
+ return entry.getKeyData();
+ }
+ return nullptr;
+}
+
+const char *
+StringPool::GetConstCStringAndSetMangledCounterPart(llvm::StringRef demangled,
+ llvm::StringRef mangled) {
+ const char *demangled_ccstr = nullptr;
+ const char *const mangled_ccstr = mangled.data();
+
+ {
+ const uint32_t demangled_hash = StringMapType::hash(demangled);
+ PoolEntry &pool = selectPool(demangled_hash);
+ std::lock_guard<PoolMutex> lock(pool.m_mutex);
+
+ // Make or update string pool entry with the mangled counterpart
+ StringMapType &map = pool.m_string_map;
+ auto [entry, inserted] =
+ map.try_emplace_with_hash(demangled, demangled_hash);
+ if (inserted)
+ pool.used_bytes += demangled.size();
+
+ entry->second = mangled_ccstr;
+
+ // Extract the const version of the demangled_cstr
+ demangled_ccstr = entry->getKeyData();
+ }
+
+ {
+ // Now assign the demangled const string as the counterpart of the
+ // mangled const string...
+ PoolEntry &pool = selectPool(mangled);
+ std::lock_guard<PoolMutex> lock(pool.m_mutex);
+ GetStringMapEntryFromKeyData(mangled_ccstr).setValue(demangled_ccstr);
+ }
+
+ // Return the constant demangled C string
+ return demangled_ccstr;
+}
+
+const char *StringPool::GetConstTrimmedCStringWithLength(const char *cstr,
+ size_t cstr_len) {
+ if (cstr != nullptr) {
+ const size_t trimmed_len = strnlen(cstr, cstr_len);
+ return GetConstCStringWithLength(cstr, trimmed_len);
+ }
+ return nullptr;
+}
+
+ConstString::MemoryStats StringPool::GetMemoryStats() const {
+ ConstString::MemoryStats stats;
+ for (size_t i = 0; i < NumPools; ++i) {
+ const PoolEntry &pool = m_string_pools[i];
+ std::shared_lock<PoolMutex> lock(pool.m_mutex);
+ const Allocator &alloc = pool.m_string_map.getAllocator();
+ stats.bytes_total += alloc.getTotalMemory();
+ stats.bytes_used += pool.used_bytes;
+ }
+ return stats;
+}
diff --git a/lldb/unittests/Utility/CMakeLists.txt b/lldb/unittests/Utility/CMakeLists.txt
index e306b7216680ec..3e3f402697f3c8 100644
--- a/lldb/unittests/Utility/CMakeLists.txt
+++ b/lldb/unittests/Utility/CMakeLists.txt
@@ -8,6 +8,7 @@ add_lldb_unittest(UtilityTests
BroadcasterTest.cpp
ChecksumTest.cpp
ConstStringTest.cpp
+ StringPoolTest.cpp
CompletionRequestTest.cpp
DataBufferTest.cpp
DataEncoderTest.cpp
diff --git a/lldb/unittests/Utility/StringPoolTest.cpp b/lldb/unittests/Utility/StringPoolTest.cpp
new file mode 100644
index 00000000000000..80d785aad9d43e
--- /dev/null
+++ b/lldb/unittests/Utility/StringPoolTest.cpp
@@ -0,0 +1,74 @@
+//===-- StringPoolTest.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Utility/StringPool.h"
+#include "gtest/gtest.h"
+
+#include <string>
+#include <thread>
+#include <vector>
+
+using namespace lldb_private;
+
+TEST(StringPoolTest, InternDeduplicates) {
+ StringPool pool;
+ std::string a = "foo";
+ const char *p1 = pool.Intern(a);
+ // The pool owns a copy, so changing the source does not affect it.
+ a[0] = 'x';
+ const char *p2 = pool.Intern("foo");
+ EXPECT_EQ(p1, p2);
+ EXPECT_STREQ("foo", p1);
+ EXPECT_NE(p1, pool.Intern("bar"));
+}
+
+TEST(StringPoolTest, InternNullAndEmpty) {
+ StringPool pool;
+ EXPECT_EQ(nullptr, pool.Intern(llvm::StringRef()));
+ const char *empty = pool.Intern("");
+ ASSERT_NE(nullptr, empty);
+ EXPECT_STREQ("", empty);
+}
+
+TEST(StringPoolTest, GlobalPoolBacksConstString) {
+ ConstString cs("global_pool_string");
+ EXPECT_EQ(cs.GetCString(),
+ StringPool::GetGlobal().Intern("global_pool_string"));
+ EXPECT_EQ(cs.GetCString(), StringPoolRef().Intern("global_pool_string"));
+}
+
+TEST(StringPoolTest, SystemPool) {
+ StringPool::Initialize();
+ EXPECT_EQ(StringPool::GetGlobal().Intern("system"),
+ StringPool::GetSystem().Intern("system"));
+ StringPool::Terminate();
+}
+
+TEST(StringPoolTest, PoolsAreIndependent) {
+ StringPool a, b;
+ EXPECT_NE(a.Intern("shared"), b.Intern("shared"));
+ EXPECT_NE(a.Intern("shared"), StringPool::GetGlobal().Intern("shared"));
+ EXPECT_EQ(5u, StringPool::GetConstCStringLength(a.Intern("12345")));
+}
+
+TEST(StringPoolTest, ConcurrentIntern) {
+ StringPool pool;
+ constexpr int num_threads = 8;
+ constexpr int num_strings = 1000;
+ std::vector<std::vector<const char *>> results(num_threads);
+ std::vector<std::thread> threads;
+ for (int t = 0; t < num_threads; ++t)
+ threads.emplace_back([&, t] {
+ for (int i = 0; i < num_strings; ++i)
+ results[t].push_back(pool.Intern(std::to_string(i)));
+ });
+ for (std::thread &t : threads)
+ t.join();
+ for (int t = 1; t < num_threads; ++t)
+ EXPECT_EQ(results[0], results[t]);
+}
>From 13b4f36ce76509b60a5b776878a1d549ba034927 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis at chromium.org>
Date: Tue, 6 Oct 2026 15:56:27 -0400
Subject: [PATCH 12/46] [clang] Don't add built-in operator candidates that
can't be viable (#227584)
If an operand of an operator has (or converts to) an arithmetic or
enumeration type, AddBuiltinOperatorCandidates() adds a built-in
candidate for every pair of promoted arithmetic types, and each
candidate tries to convert both operands. That's 121 candidates for `a
== b` on arm64 macOS.
If an operand can't convert to any of these types, none of these
candidates are viable (e.g. `e == E::kFoo` for a scoped enumeration E).
So conservatively track if each operand can convert to an arithmetic
type, and only add these candidates if all operands can convert.
Non-viable built-in candidates aren't used for anything and are safe to
not add.
For 60 random Chromium TUs (linux x64, -O2) picked with probability
proportional to their compile time, sum over all TUs:
CPU time: 186.6 s => 183.3 s, -1.81%
instructions: 1901.2e9 => 1782.9e9, -6.23%
mean max RSS: 587.7 MB => 587.0 MB (noise)
(To check noise level, running this with the same binary twice: -0.03%
CPU time, -0.01% instructions, +0.21% RSS.)
Every TU improves, by 2.1% to 10.1% instructions.
No intended behavior change.
---
clang/lib/Sema/SemaOverload.cpp | 51 +++++++++----
...d-builtin-operators-arithmetic-pruning.cpp | 71 +++++++++++++++++++
2 files changed, 108 insertions(+), 14 deletions(-)
create mode 100644 clang/test/SemaCXX/overloaded-builtin-operators-arithmetic-pruning.cpp
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index cd1bd2cecc4908..e5049d11480ac4 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -9197,6 +9197,12 @@ class BuiltinCandidateTypeSet {
/// were present in the candidate set.
bool HasArithmeticOrEnumeralTypes;
+ /// A flag indicating whether the candidate set has a type that might
+ /// convert to a promoted arithmetic type or to a vector type. This is
+ /// conservative: only scoped enumerations, pointers, member pointers,
+ /// nullptr_t, and classes that convert to nothing else are known not to.
+ bool MayConvertToArithmetic;
+
/// A flag indicating whether the nullptr type was present in the
/// candidate set.
bool HasNullPtrType;
@@ -9222,8 +9228,8 @@ class BuiltinCandidateTypeSet {
BuiltinCandidateTypeSet(Sema &SemaRef)
: HasNonRecordTypes(false), HasArithmeticOrEnumeralTypes(false),
- HasNullPtrType(false), HasReflectionType(false), SemaRef(SemaRef),
- Context(SemaRef.Context) {}
+ MayConvertToArithmetic(false), HasNullPtrType(false),
+ HasReflectionType(false), SemaRef(SemaRef), Context(SemaRef.Context) {}
void AddTypesConvertedFrom(QualType Ty,
SourceLocation Loc,
@@ -9245,6 +9251,7 @@ class BuiltinCandidateTypeSet {
bool containsMatrixType(QualType Ty) const { return MatrixTypes.count(Ty); }
bool hasNonRecordTypes() { return HasNonRecordTypes; }
bool hasArithmeticOrEnumeralTypes() { return HasArithmeticOrEnumeralTypes; }
+ bool mayConvertToArithmetic() const { return MayConvertToArithmetic; }
bool hasNullPtrType() const { return HasNullPtrType; }
bool hasReflectionType() const { return HasReflectionType; }
};
@@ -9399,6 +9406,13 @@ BuiltinCandidateTypeSet::AddTypesConvertedFrom(QualType Ty,
HasArithmeticOrEnumeralTypes =
HasArithmeticOrEnumeralTypes || Ty->isArithmeticType();
+ // Flag if the type might convert to a promoted arithmetic or vector type.
+ // For records, this is set below when visiting their conversion functions.
+ MayConvertToArithmetic =
+ MayConvertToArithmetic ||
+ !(TyIsRec || Ty->isScopedEnumeralType() || Ty->isAnyPointerType() ||
+ Ty->isMemberPointerType() || Ty->isNullPtrType());
+
if (Ty->isObjCIdType() || Ty->isObjCClassType())
PointerTypes.insert(Ty);
else if (Ty->getAs<PointerType>() || Ty->getAs<ObjCObjectPointerType>()) {
@@ -9442,8 +9456,10 @@ BuiltinCandidateTypeSet::AddTypesConvertedFrom(QualType Ty,
// Skip conversion function templates; they don't tell us anything
// about which builtin types we can convert to.
- if (isa<FunctionTemplateDecl>(D))
+ if (isa<FunctionTemplateDecl>(D)) {
+ MayConvertToArithmetic = true;
continue;
+ }
CXXConversionDecl *Conv = cast<CXXConversionDecl>(D);
if (AllowExplicitConversions || !Conv->isExplicit()) {
@@ -9590,7 +9606,10 @@ class BuiltinOperatorOverloadBuilder {
Sema &S;
ArrayRef<Expr *> Args;
QualifiersAndAtomic VisibleTypeConversionsQuals;
- bool HasArithmeticOrEnumeralCandidateType;
+ // Whether a candidate whose parameters are all arithmetic, vector or matrix
+ // types can be viable. It is viable if there is an arithmetic or enumeral
+ // candidate type, and every argument might convert to such a type.
+ bool ArithmeticCandidatesMayBeViable;
SmallVectorImpl<BuiltinCandidateTypeSet> &CandidateTypes;
OverloadCandidateSet &CandidateSet;
@@ -9736,8 +9755,12 @@ class BuiltinOperatorOverloadBuilder {
OverloadCandidateSet &CandidateSet)
: S(S), Args(Args),
VisibleTypeConversionsQuals(VisibleTypeConversionsQuals),
- HasArithmeticOrEnumeralCandidateType(
- HasArithmeticOrEnumeralCandidateType),
+ ArithmeticCandidatesMayBeViable(
+ HasArithmeticOrEnumeralCandidateType &&
+ llvm::all_of(CandidateTypes,
+ [](const BuiltinCandidateTypeSet &Types) {
+ return Types.mayConvertToArithmetic();
+ })),
CandidateTypes(CandidateTypes), CandidateSet(CandidateSet) {
InitArithmeticTypes();
}
@@ -9762,7 +9785,7 @@ class BuiltinOperatorOverloadBuilder {
// VQ T& operator--(VQ T&);
// T operator--(VQ T&, int);
void addPlusPlusMinusMinusArithmeticOverloads(OverloadedOperatorKind Op) {
- if (!HasArithmeticOrEnumeralCandidateType)
+ if (!ArithmeticCandidatesMayBeViable)
return;
for (unsigned Arith = 0; Arith < NumArithmeticTypes; ++Arith) {
@@ -9836,7 +9859,7 @@ class BuiltinOperatorOverloadBuilder {
// T operator+(T);
// T operator-(T);
void addUnaryPlusOrMinusArithmeticOverloads() {
- if (!HasArithmeticOrEnumeralCandidateType)
+ if (!ArithmeticCandidatesMayBeViable)
return;
for (unsigned Arith = FirstPromotedArithmeticType;
@@ -9866,7 +9889,7 @@ class BuiltinOperatorOverloadBuilder {
//
// T operator~(T);
void addUnaryTildePromotedIntegralOverloads() {
- if (!HasArithmeticOrEnumeralCandidateType)
+ if (!ArithmeticCandidatesMayBeViable)
return;
for (unsigned Int = FirstPromotedIntegralType;
@@ -10089,7 +10112,7 @@ class BuiltinOperatorOverloadBuilder {
// between types L and R.
// Our candidates ignore the first parameter.
void addGenericBinaryArithmeticOverloads() {
- if (!HasArithmeticOrEnumeralCandidateType)
+ if (!ArithmeticCandidatesMayBeViable)
return;
for (unsigned Left = FirstPromotedArithmeticType;
@@ -10117,7 +10140,7 @@ class BuiltinOperatorOverloadBuilder {
/// * (M2.getElementType(), M2) -> M2
/// * (M2, M2) -> M2 // Only if M2 is not part of CandidateTypes[0].
void addMatrixBinaryArithmeticOverloads() {
- if (!HasArithmeticOrEnumeralCandidateType)
+ if (!ArithmeticCandidatesMayBeViable)
return;
for (QualType M1 : CandidateTypes[0].matrix_types()) {
@@ -10182,7 +10205,7 @@ class BuiltinOperatorOverloadBuilder {
// where LR is the result of the usual arithmetic conversions
// between types L and R.
void addBinaryBitwiseArithmeticOverloads() {
- if (!HasArithmeticOrEnumeralCandidateType)
+ if (!ArithmeticCandidatesMayBeViable)
return;
for (unsigned Left = FirstPromotedIntegralType;
@@ -10347,7 +10370,7 @@ class BuiltinOperatorOverloadBuilder {
// VQ L& operator+=(VQ L&, R);
// VQ L& operator-=(VQ L&, R);
void addAssignmentArithmeticOverloads(bool isEqualOp) {
- if (!HasArithmeticOrEnumeralCandidateType)
+ if (!ArithmeticCandidatesMayBeViable)
return;
for (unsigned Left = 0; Left < NumArithmeticTypes; ++Left) {
@@ -10401,7 +10424,7 @@ class BuiltinOperatorOverloadBuilder {
// VQ L& operator^=(VQ L&, R);
// VQ L& operator|=(VQ L&, R);
void addAssignmentIntegralOverloads() {
- if (!HasArithmeticOrEnumeralCandidateType)
+ if (!ArithmeticCandidatesMayBeViable)
return;
for (unsigned Left = FirstIntegralType; Left < LastIntegralType; ++Left) {
diff --git a/clang/test/SemaCXX/overloaded-builtin-operators-arithmetic-pruning.cpp b/clang/test/SemaCXX/overloaded-builtin-operators-arithmetic-pruning.cpp
new file mode 100644
index 00000000000000..249e427bd4a5f5
--- /dev/null
+++ b/clang/test/SemaCXX/overloaded-builtin-operators-arithmetic-pruning.cpp
@@ -0,0 +1,71 @@
+// RUN: %clang_cc1 -fsyntax-only -std=c++20 -verify %s
+
+// The built-in operator candidates that only have arithmetic or vector
+// parameter types are only added if every operand might convert to such a
+// type. Check that this doesn't lose candidates.
+
+enum class Scoped { A };
+enum Unscoped { UA };
+typedef int Vec __attribute__((vector_size(16)));
+
+struct ConvTemplate { template <class T> operator T() const; };
+struct ConvInt { operator int() const; };
+struct ConvConvInt { operator ConvInt() const; };
+struct ConvIntRef { operator int &() const; };
+struct ConvScoped { operator Scoped() const; };
+struct ConvUnscoped { operator Unscoped() const; };
+struct ConvPtr { operator int *() const; };
+struct ConvVec { operator Vec() const; };
+struct ExplicitBool { explicit operator bool() const; };
+
+Scoped operator|(Scoped, Scoped);
+
+void test(Scoped S, Unscoped U, ConvTemplate CT, ConvInt CI, ConvIntRef CIR,
+ ConvConvInt CCI, ConvScoped CS, ConvUnscoped CU, ConvPtr CP, ConvVec CV,
+ ExplicitBool EB, int I, int *P, Vec V, _Atomic(int) AI) {
+ // Conversion function templates might convert to anything.
+ (void)(CT == 1); // expected-error {{use of overloaded operator '==' is ambiguous (with operand types 'ConvTemplate' and 'int')}} \
+ // expected-note-re 1+ {{built-in candidate operator==({{.*}}, int)}}
+ (void)(1 - CT); // expected-error {{use of overloaded operator '-' is ambiguous (with operand types 'int' and 'ConvTemplate')}} \
+ // expected-note-re 1+ {{built-in candidate operator-(int, {{.*}})}}
+
+ (void)(CI == 1);
+ (void)(CI + CI);
+ (void)(CI << CU);
+ (void)(-CI);
+ (void)(~CU);
+ (void)(U + CI);
+ (void)(U | U);
+ (void)(CI ? U : CI);
+ CIR += CI;
+ CIR |= CU;
+ ++CIR;
+ I += CI;
+ I <<= CU;
+ AI += CI;
+ AI |= CI;
+ (void)(AI == CI);
+ (void)(CP == P);
+ (void)(CP + CI);
+ (void)(CV + CV);
+ (void)(CV == V);
+ V += CV;
+
+ (void)(S == Scoped::A);
+ (void)(S | S);
+ (void)(CS == Scoped::A);
+
+ (void)(S == 1); // expected-error {{invalid operands to binary expression ('Scoped' and 'int')}} \
+ // expected-note {{no implicit conversion for scoped enum}}
+ (void)(S + CI); // expected-error {{invalid operands to binary expression ('Scoped' and 'ConvInt')}}
+ (void)(CS == 1); // expected-error {{invalid operands to binary expression ('ConvScoped' and 'int')}}
+ (void)(CCI == 1); // expected-error {{invalid operands to binary expression ('ConvConvInt' and 'int')}}
+ (void)(CP == 1); // expected-error {{invalid operands to binary expression ('ConvPtr' and 'int')}}
+ (void)(EB == 1); // expected-error {{invalid operands to binary expression ('ExplicitBool' and 'int')}}
+ (void)(AI + S); // expected-error {{invalid operands to binary expression ('_Atomic(int)' and 'Scoped')}} \
+ // expected-note {{no implicit conversion for scoped enum}}
+ I += S; // expected-error {{invalid operands to binary expression ('int' and 'Scoped')}} \
+ // expected-note {{no implicit conversion for scoped enum}}
+ (void)(CV + S); // expected-error {{invalid operands to binary expression ('ConvVec' and 'Scoped')}}
+ (void)(-S); // expected-error {{invalid argument type 'Scoped' to unary expression}}
+}
>From 10490c94eb1dde814f4ced5cd2d71b4770ea29ca Mon Sep 17 00:00:00 2001
From: Vineet Kumar <173554+vntkmr at users.noreply.github.com>
Date: Tue, 6 Oct 2026 16:07:12 -0400
Subject: [PATCH 13/46] [flang][Semantics] Enforce F2023 C1545 only for generic
references (#219581)
F2023 C1545 requires the consequent-args of a conditional argument to
agree on corank and on the ALLOCATABLE and POINTER attributes, but only
in a reference to a generic procedure. Flang applied the constraint to
every reference containing a conditional argument and rejected
conforming
code.
Move the cross-consequent checks out of CheckConditionalArg() and run
them from CheckAndResolveGenericReference(), before ResolveGeneric()
and only when the reference is generic. C1545 is a constraint, not one
of the generic resolution rules in F2023 15.5.5, so check it separately
from candidate matching.
When C1545 fails, return a GenericResolution with errorReported set
without calling ResolveGeneric(). The callers stop analyzing the
reference without additional generic-resolution diagnostics.
GetCalleeAndArguments() also returns before fallback intrinsic probing.
The parse-tree symbol stays generic, so a second analysis of the
expression reports the violation again.
Add IntrinsicProcTable::IsGenericIntrinsic() to distinguish generic
intrinsic functions and subroutines from specific intrinsic names such
as DSIN, which also carry Attr::INTRINSIC.
Use IsObjectPointer() for the POINTER check, matching generic
resolution.
This includes references to functions that return object pointers.
Pass each argument's optional source location to the C1545 diagnostics.
Conditional arguments do not yet carry source locations, so these calls
retain the enclosing-call fallback. Issue llvm/llvm-project#227805
tracks
the separate source-range capture and propagation work.
Tests cover specific and generic references, intrinsic functions and
subroutines, type-bound calls, pointer-valued function references, and
repeated analysis inside array constructors with constant implied-DO
bounds. A MAX call with a missing second argument checks that a C1545
violation prevents further intrinsic probing.
These changes were generated with the assistance of AI tooling and have
been reviewed, tested, and validated by the author.
Fixes llvm/llvm-project#209840
---
flang/include/flang/Evaluate/intrinsics.h | 5 +
flang/include/flang/Semantics/expression.h | 14 ++
flang/lib/Evaluate/intrinsics.cpp | 16 ++
flang/lib/Semantics/check-call.cpp | 117 +++++----
flang/lib/Semantics/check-call.h | 8 +
flang/lib/Semantics/expression.cpp | 39 ++-
flang/test/Semantics/conditional-arg.f90 | 267 ++++++++++++++++++++-
7 files changed, 399 insertions(+), 67 deletions(-)
diff --git a/flang/include/flang/Evaluate/intrinsics.h b/flang/include/flang/Evaluate/intrinsics.h
index 1757dc2ed24677..bb41cbec657c1c 100644
--- a/flang/include/flang/Evaluate/intrinsics.h
+++ b/flang/include/flang/Evaluate/intrinsics.h
@@ -88,6 +88,11 @@ class IntrinsicProcTable {
bool IsIntrinsicSubroutine(const std::string &) const;
bool IsDualIntrinsic(const std::string &) const;
+ // True when a reference to this name is a reference to a generic procedure.
+ // A name that is only a specific intrinsic name, such as DSIN, is not; a
+ // name that is both, such as SIN, is.
+ bool IsGenericIntrinsic(const std::string &) const;
+
// Inquiry intrinsics are defined in section 16.7, table 16.1
IntrinsicClass GetIntrinsicClass(const std::string &) const;
diff --git a/flang/include/flang/Semantics/expression.h b/flang/include/flang/Semantics/expression.h
index 59cd424c4dae97..bcee8d65b58e25 100644
--- a/flang/include/flang/Semantics/expression.h
+++ b/flang/include/flang/Semantics/expression.h
@@ -375,10 +375,24 @@ class ExpressionAnalyzer {
const Symbol *specific{nullptr};
bool failedDueToAmbiguity{false};
SymbolVector tried{};
+ // Resolution was not attempted because the reference is already in error;
+ // callers must not report a resolution failure too.
+ bool errorReported{false};
};
+ // Resolves a generic identifier against an argument list. Does not report
+ // resolution failure; callers do that with EmitGenericResolutionError. It
+ // is not side-effect free, though: ResolveForward can emit a message and can
+ // set a durable error flag, so speculative callers buffer messages around it.
GenericResolution ResolveGeneric(const Symbol &, const ActualArguments &,
const AdjustActuals &, bool isSubroutine, SymbolVector &&tried,
bool mightBeStructureConstructor = false);
+ // Use this rather than ResolveGeneric for a generic reference that appears in
+ // the source: it also applies the constraints that hold only for such a
+ // reference (F2023 C1545), so it must not be called speculatively. A
+ // violation sets errorReported and skips resolution.
+ GenericResolution CheckAndResolveGenericReference(const Symbol &,
+ const ActualArguments &, const AdjustActuals &, bool isSubroutine,
+ SymbolVector &&tried, bool mightBeStructureConstructor = false);
void EmitGenericResolutionError(const Symbol &, bool dueToNullActuals,
bool isSubroutine, const ActualArguments &, const SymbolVector &,
const AdjustActuals &);
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index e6f39ae89ab053..70561b6caaf232 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -2928,6 +2928,7 @@ class IntrinsicProcTable::Implementation {
bool IsIntrinsicFunction(const std::string &) const;
bool IsIntrinsicSubroutine(const std::string &) const;
bool IsDualIntrinsic(const std::string &) const;
+ bool IsGenericIntrinsic(const std::string &) const;
IntrinsicClass GetIntrinsicClass(const std::string &) const;
std::string GetGenericIntrinsicName(const std::string &) const;
@@ -3007,6 +3008,18 @@ bool IntrinsicProcTable::Implementation::IsIntrinsic(
const std::string &name) const {
return IsIntrinsicFunction(name) || IsIntrinsicSubroutine(name);
}
+bool IntrinsicProcTable::Implementation::IsGenericIntrinsic(
+ const std::string &name0) const {
+ const std::string &name{ResolveAlias(name0)};
+ // Intrinsic subroutines have no specific names, so a reference to one is
+ // always a reference to a generic procedure.
+ // Unlike IsIntrinsicFunction() and IsIntrinsicSubroutine(), the names that
+ // Probe() special-cases ahead of these tables are deliberately omitted here:
+ // each of them has a single interface, so nothing about such a reference can
+ // depend on which arguments it is given.
+ return genericFuncs_.find(name) != genericFuncs_.end() ||
+ subroutines_.find(name) != subroutines_.end();
+}
bool IntrinsicProcTable::Implementation::IsDualIntrinsic(
const std::string &name) const {
// Collection for some intrinsics with function and subroutine form,
@@ -4465,6 +4478,9 @@ bool IntrinsicProcTable::IsIntrinsicSubroutine(const std::string &name) const {
bool IntrinsicProcTable::IsDualIntrinsic(const std::string &name) const {
return DEREF(impl_.get()).IsDualIntrinsic(name);
}
+bool IntrinsicProcTable::IsGenericIntrinsic(const std::string &name) const {
+ return DEREF(impl_.get()).IsGenericIntrinsic(name);
+}
IntrinsicClass IntrinsicProcTable::GetIntrinsicClass(
const std::string &name) const {
diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp
index 508fc079815084..feabf0cd33bf09 100644
--- a/flang/lib/Semantics/check-call.cpp
+++ b/flang/lib/Semantics/check-call.cpp
@@ -1512,8 +1512,9 @@ static void CheckProcedureArg(evaluate::ActualArgument &arg,
}
}
-// F2023 C1540-C1543, C1545: Check conditional argument against dummy data
-// object
+// F2023 C1540-C1543: Check a conditional argument against a dummy data object.
+// C1544 is absent by design: it is a grammar disambiguation rule that the
+// parser satisfies automatically, see "consequent" in program-parsers.cpp.
static void CheckConditionalArg(
const evaluate::ActualArgument::ConditionalArg &condArg,
const characteristics::DummyDataObject &object,
@@ -1554,54 +1555,69 @@ static void CheckConditionalArg(
"Each consequent-arg in conditional argument associated with a coarray %s must be a coarray"_err_en_US,
dummyName);
}
- // C1544: the requirement that each consequent-arg match the dummy's
- // ALLOCATABLE/POINTER attribute is enforced by the standard
- // explicit-interface check (checkOneExpr) run on each non-.NIL.
- // consequent below.
}};
condArg.ForEachConsequent(checkOneConsequent);
- // C1545: in a reference to a generic procedure, each consequent-arg shall
- // have the same corank, and if any has the ALLOCATABLE or POINTER attribute,
- // each shall have it. Strictly, this requirement applies only to references
- // to generic procedures, where it avoids ambiguity when resolving the generic
- // to a specific procedure; for a specific procedure reference these
- // combinations are otherwise allowed. For now it is enforced unconditionally
- // here.
- // TODO: move this check into generic resolution (ResolveGeneric) and enforce
- // it precisely, i.e. only for references to generic procedures, where the
- // ambiguity it guards against can actually arise.
- std::optional<int> firstCorank;
- std::optional<bool> firstIsAllocatable;
- std::optional<bool> firstIsPointer;
- auto checkConsistency{[&](const evaluate::ActualArgument::ConditionalArg::
- Consequent &cons) {
- if (!cons) {
- return;
- }
- auto &consExpr{cons->value()};
- int corank{evaluate::GetCorank(consExpr)};
- bool isAlloc{evaluate::IsAllocatableDesignator(consExpr)};
- bool isPtr{evaluate::IsObjectPointer(consExpr)};
- if (!firstCorank) {
- firstCorank = corank;
- firstIsAllocatable = isAlloc;
- firstIsPointer = isPtr;
- } else {
- if (corank != *firstCorank) {
- messages.Say(
- "All consequent-args in a conditional argument must have the same corank"_err_en_US);
- }
- if (isAlloc != *firstIsAllocatable) {
- messages.Say(
- "If any consequent-arg in a conditional argument has the ALLOCATABLE attribute, each must have it"_err_en_US);
- }
- if (isPtr != *firstIsPointer) {
- messages.Say(
- "If any consequent-arg in a conditional argument has the POINTER attribute, each must have it"_err_en_US);
- }
- }
- }};
- condArg.ForEachConsequent(checkConsistency);
+}
+
+// F2023 C1545: in a reference to a generic procedure, each consequent-arg
+// shall have the same corank, and if any consequent-arg has the ALLOCATABLE
+// or POINTER attribute, each consequent-arg shall have that attribute.
+// These properties decide which specific procedures an actual argument may be
+// associated with, so letting them differ would make generic resolution depend
+// on a condition whose value is not known until run time. A reference to a
+// specific procedure has nothing to resolve, and these combinations are
+// allowed there.
+bool CheckConditionalArgsInGenericReference(
+ const evaluate::ActualArguments &actuals,
+ parser::ContextualMessages &messages) {
+ bool anyReported{false};
+ for (const std::optional<evaluate::ActualArgument> &arg : actuals) {
+ const auto *condArg{arg ? arg->GetConditionalArg() : nullptr};
+ if (!condArg) {
+ continue;
+ }
+ const auto *first{condArg->FirstNonNilConsequent()};
+ if (!first) { // every consequent-arg is .NIL.
+ continue;
+ }
+ const int firstCorank{evaluate::GetCorank(*first)};
+ const bool firstIsAllocatable{evaluate::IsAllocatableDesignator(*first)};
+ // Use the same POINTER test as generic resolution, which accepts a
+ // pointer-valued function reference.
+ const bool firstIsPointer{evaluate::IsObjectPointer(*first)};
+ bool corankDiffers{false};
+ bool allocatableDiffers{false};
+ bool pointerDiffers{false};
+ // Comparing the first consequent-arg against itself is a no-op, so it needs
+ // no special case here.
+ condArg->ForEachConsequent(
+ [&](const evaluate::ActualArgument::ConditionalArg::Consequent &cons) {
+ if (!cons) { // .NIL. has no corank or attributes to compare
+ return;
+ }
+ const auto &consExpr{cons->value()};
+ corankDiffers |= evaluate::GetCorank(consExpr) != firstCorank;
+ allocatableDiffers |=
+ evaluate::IsAllocatableDesignator(consExpr) != firstIsAllocatable;
+ pointerDiffers |=
+ evaluate::IsObjectPointer(consExpr) != firstIsPointer;
+ });
+ // Report each violated requirement once, however long the chain is.
+ if (corankDiffers) {
+ messages.Say(arg->sourceLocation(),
+ "In a reference to a generic procedure, all consequent-args in a conditional argument must have the same corank"_err_en_US);
+ }
+ if (allocatableDiffers) {
+ messages.Say(arg->sourceLocation(),
+ "In a reference to a generic procedure, if any consequent-arg in a conditional argument has the ALLOCATABLE attribute, each must have it"_err_en_US);
+ }
+ if (pointerDiffers) {
+ messages.Say(arg->sourceLocation(),
+ "In a reference to a generic procedure, if any consequent-arg in a conditional argument has the POINTER attribute, each must have it"_err_en_US);
+ }
+ anyReported |= corankDiffers || allocatableDiffers || pointerDiffers;
+ }
+ return anyReported;
}
// Allow BOZ literal actual arguments when they can be converted to a known
@@ -1723,8 +1739,9 @@ static void CheckExplicitInterfaceArg(evaluate::ActualArgument &arg,
}};
if (auto *condArg{arg.GetConditionalArg()}) {
CheckConditionalArg(*condArg, object, dummyName, messages);
- // Also run standard explicit-interface checks on each
- // non-.NIL. consequent expression (recursive).
+ // Each non-.NIL. consequent-arg is itself an actual argument,
+ // so these checks are what enforce the dummy's ALLOCATABLE and
+ // POINTER requirements (F2023 15.5.2.6, 15.5.2.7).
condArg->ForEachConsequent(
[&](evaluate::ActualArgument::ConditionalArg::Consequent
&cons) {
diff --git a/flang/lib/Semantics/check-call.h b/flang/lib/Semantics/check-call.h
index cdfecd58423a7d..8caf71479fe5aa 100644
--- a/flang/lib/Semantics/check-call.h
+++ b/flang/lib/Semantics/check-call.h
@@ -64,5 +64,13 @@ parser::Messages CheckExplicitInterface(
bool CheckInterfaceForGeneric(const evaluate::characteristics::Procedure &,
evaluate::ActualArguments &, SemanticsContext &,
bool allowActualArgumentConversions = false);
+
+// F2023 C1545: reports the cross-consequent-arg inconsistencies that a
+// conditional argument may not have in a reference to a generic procedure, and
+// returns whether any were reported. C1545 does not apply to a reference to a
+// specific procedure, though the ordinary argument-association rules may still
+// reject the same combinations there.
+bool CheckConditionalArgsInGenericReference(
+ const evaluate::ActualArguments &, parser::ContextualMessages &);
} // namespace Fortran::semantics
#endif
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index e3ed0e0be52d4b..6448c35b922ffc 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -2795,12 +2795,14 @@ auto ExpressionAnalyzer::AnalyzeProcedureComponentRef(
}
return true;
}};
- auto result{ResolveGeneric(
+ auto result{CheckAndResolveGenericReference(
generic, arguments, adjustment, isSubroutine, SymbolVector{})};
sym = result.specific;
if (!sym) {
- EmitGenericResolutionError(generic, result.failedDueToAmbiguity,
- isSubroutine, arguments, result.tried, adjustment);
+ if (!result.errorReported) {
+ EmitGenericResolutionError(generic, result.failedDueToAmbiguity,
+ isSubroutine, arguments, result.tried, adjustment);
+ }
return std::nullopt;
}
// re-resolve the name to the specific binding
@@ -3440,6 +3442,29 @@ auto ExpressionAnalyzer::ResolveGeneric(const Symbol &symbol,
return {nullptr, false, std::move(tried)};
}
+auto ExpressionAnalyzer::CheckAndResolveGenericReference(const Symbol &symbol,
+ const ActualArguments &actuals, const AdjustActuals &adjustActuals,
+ bool isSubroutine, SymbolVector &&tried, bool mightBeStructureConstructor)
+ -> GenericResolution {
+ const Symbol &ultimate{symbol.GetUltimate()};
+ // Attr::INTRINSIC is set for specific intrinsic names such as DSIN as well,
+ // and a reference to one of those is not a reference to a generic procedure.
+ if ((ultimate.has<semantics::GenericDetails>() ||
+ (ultimate.attrs().test(semantics::Attr::INTRINSIC) &&
+ context_.intrinsics().IsGenericIntrinsic(
+ ultimate.name().ToString()))) &&
+ semantics::CheckConditionalArgsInGenericReference(
+ actuals, GetContextualMessages())) {
+ // Not resolving also leaves the parse tree symbol generic, so a second
+ // analysis of the expression reports the violation again.
+ GenericResolution result;
+ result.errorReported = true;
+ return result;
+ }
+ return ResolveGeneric(symbol, actuals, adjustActuals, isSubroutine,
+ std::move(tried), mightBeStructureConstructor);
+}
+
const Symbol &ExpressionAnalyzer::AccessSpecific(
const Symbol &originalGeneric, const Symbol &specific) {
if (const auto *hosted{
@@ -3557,8 +3582,12 @@ auto ExpressionAnalyzer::GetCalleeAndArguments(const parser::Name &name,
SymbolVector tried;
if (isGenericInterface || isExplicitIntrinsic) {
ExpressionAnalyzer::AdjustActuals noAdjustment;
- auto result{ResolveGeneric(*symbol, arguments, noAdjustment, isSubroutine,
- SymbolVector{}, mightBeStructureConstructor)};
+ auto result{
+ CheckAndResolveGenericReference(*symbol, arguments, noAdjustment,
+ isSubroutine, SymbolVector{}, mightBeStructureConstructor)};
+ if (result.errorReported) {
+ return std::nullopt;
+ }
resolution = result.specific;
dueToAmbiguity = result.failedDueToAmbiguity;
tried = std::move(result.tried);
diff --git a/flang/test/Semantics/conditional-arg.f90 b/flang/test/Semantics/conditional-arg.f90
index fac95dd6dc4ed6..d0fa05de8ed83c 100644
--- a/flang/test/Semantics/conditional-arg.f90
+++ b/flang/test/Semantics/conditional-arg.f90
@@ -68,6 +68,51 @@ pure integer function func_two(x, y)
end function
end interface
+ ! Generic identifiers, used by the F2023 C1545 tests below. C1545 applies
+ ! only to a reference to a generic procedure.
+ interface gen_plain
+ subroutine gen_plain_int(x)
+ integer, intent(in) :: x
+ end subroutine
+ subroutine gen_plain_real(x)
+ real, intent(in) :: x
+ end subroutine
+ end interface
+
+ interface gen_opt
+ subroutine gen_opt_int(x)
+ integer, intent(in), optional :: x
+ end subroutine
+ subroutine gen_opt_real(x)
+ real, intent(in), optional :: x
+ end subroutine
+ end interface
+
+ interface gen_func
+ pure integer function gen_func_int(x)
+ integer, intent(in) :: x
+ end function
+ pure real function gen_func_real(x)
+ real, intent(in) :: x
+ end function
+ end interface
+
+ interface
+ function func_ptr_result()
+ integer, pointer :: func_ptr_result
+ end function
+ end interface
+
+ type :: t_gen
+ contains
+ procedure, nopass :: tbp_gen_int
+ procedure, nopass :: tbp_gen_real
+ procedure, nopass :: tbp_fn_int
+ procedure, nopass :: tbp_fn_real
+ generic :: gen => tbp_gen_int, tbp_gen_real
+ generic :: gen_fn => tbp_fn_int, tbp_fn_real
+ end type
+
contains
! =========================================================================
@@ -424,38 +469,236 @@ subroutine test_typeless_null_both_consequents
! shall have the same corank, and if any consequent-arg has the
! ALLOCATABLE or POINTER attribute, each consequent-arg shall
! have that attribute.
+ !
+ ! The constraint is scoped to generic references, where these
+ ! attributes decide which specific procedures the actual
+ ! argument may associate with. A reference to a specific
+ ! procedure has nothing to resolve and is not constrained.
! =========================================================================
- subroutine test_f2023_c1545_allocatable_inconsistency
+ ! Bindings for the type-bound generic 'gen' of t_gen.
+ subroutine tbp_gen_int(x)
+ integer, intent(in) :: x
+ end subroutine
+
+ subroutine tbp_gen_real(x)
+ real, intent(in) :: x
+ end subroutine
+
+ pure integer function tbp_fn_int(x)
+ integer, intent(in) :: x
+ tbp_fn_int = x
+ end function
+
+ pure real function tbp_fn_real(x)
+ real, intent(in) :: x
+ tbp_fn_real = x
+ end function
+
+ subroutine test_f2023_c1545_generic_allocatable_inconsistency
integer, allocatable :: a
integer :: b
logical :: flag
- !ERROR: If any consequent-arg in a conditional argument has the ALLOCATABLE attribute, each must have it
- !ERROR: ALLOCATABLE dummy argument 'x=' must be associated with an ALLOCATABLE actual argument
- call sub_alloc((flag ? a : b))
+ !ERROR: In a reference to a generic procedure, if any consequent-arg in a conditional argument has the ALLOCATABLE attribute, each must have it
+ call gen_plain((flag ? a : b))
end subroutine
- subroutine test_f2023_c1545_pointer_inconsistency
+ subroutine test_f2023_c1545_generic_pointer_inconsistency
integer, pointer :: p
integer :: b
logical :: flag
- !ERROR: If any consequent-arg in a conditional argument has the POINTER attribute, each must have it
- !ERROR: Actual argument associated with POINTER dummy argument 'x=' must also be POINTER unless INTENT(IN)
- call sub_pointer((flag ? p : b))
+ !ERROR: In a reference to a generic procedure, if any consequent-arg in a conditional argument has the POINTER attribute, each must have it
+ call gen_plain((flag ? p : b))
+ end subroutine
+
+ subroutine test_f2023_c1545_generic_corank_inconsistency(c, d, flag)
+ integer, intent(in) :: c[*], d
+ logical, intent(in) :: flag
+ !ERROR: In a reference to a generic procedure, all consequent-args in a conditional argument must have the same corank
+ call gen_plain((flag ? c : d))
+ end subroutine
+
+ subroutine test_f2023_c1545_generic_corank_consistent(c, d, flag)
+ integer, intent(in) :: c[*], d[*]
+ logical, intent(in) :: flag
+ ! Both consequent-args have corank 1 -- no C1545 error
+ call gen_plain((flag ? c : d))
+ end subroutine
+
+ subroutine test_f2023_c1545_generic_three_way(f1, f2)
+ logical :: f1, f2
+ integer, allocatable :: c
+ integer :: a, b
+ ! The offending consequent-arg is in the tail of the chain
+ !ERROR: In a reference to a generic procedure, if any consequent-arg in a conditional argument has the ALLOCATABLE attribute, each must have it
+ call gen_plain((f1 ? a : f2 ? b : c))
+ end subroutine
+
+ ! .NIL. is not an entity, so it neither seeds nor participates in the C1545
+ ! comparison; the mismatch between the remaining two is still reported.
+ subroutine test_f2023_c1545_generic_nil_ignored(f1, f2)
+ logical :: f1, f2
+ integer, allocatable :: a
+ integer :: b
+ !ERROR: In a reference to a generic procedure, if any consequent-arg in a conditional argument has the ALLOCATABLE attribute, each must have it
+ call gen_opt((f1 ? .NIL. : f2 ? a : b))
+ end subroutine
+
+ subroutine test_f2023_c1545_generic_nil_consistent(f1, f2)
+ logical :: f1, f2
+ integer, allocatable :: a, b
+ ! Both non-.NIL. consequent-args are allocatable -- no C1545 error
+ call gen_opt((f1 ? .NIL. : f2 ? a : b))
+ end subroutine
+
+ ! A specific type-bound procedure is not a generic reference.
+ subroutine test_c1545_specific_type_bound
+ type(t_gen) :: obj
+ integer, allocatable :: a
+ integer :: b
+ logical :: flag
+ call obj%tbp_gen_int((flag ? a : b))
+ end subroutine
+
+ ! Constant implied-DO bounds make the array constructor body be analyzed a
+ ! second time; the diagnostic must survive that.
+ subroutine test_f2023_c1545_generic_in_array_constructor
+ integer, allocatable :: a
+ integer :: b, i
+ integer :: arr(3)
+ logical :: flag
+ !ERROR: In a reference to a generic procedure, if any consequent-arg in a conditional argument has the ALLOCATABLE attribute, each must have it
+ arr = [(gen_func((flag ? a : b)), i=1,3)]
+ end subroutine
+
+ ! Same, for a type-bound generic: it is re-resolved on a different symbol.
+ subroutine test_f2023_c1545_type_bound_generic_in_array_constructor
+ type(t_gen) :: obj
+ integer, allocatable :: a
+ integer :: b, i
+ integer :: arr(3)
+ logical :: flag
+ !ERROR: In a reference to a generic procedure, if any consequent-arg in a conditional argument has the ALLOCATABLE attribute, each must have it
+ arr = [(obj%gen_fn((flag ? a : b)), i=1,3)]
+ end subroutine
+
+ ! A pointer-valued function reference has the POINTER attribute here, as it
+ ! does in generic resolution, so mixing it with a non-pointer violates C1545.
+ subroutine test_c1545_pointer_function_result_differs
+ integer :: b, r
+ logical :: flag
+ !ERROR: In a reference to a generic procedure, if any consequent-arg in a conditional argument has the POINTER attribute, each must have it
+ r = gen_func((flag ? func_ptr_result() : b))
+ end subroutine
+
+ ! gen_func(p) and gen_func(func_ptr_result()) resolve to the same specific,
+ ! so the conditional argument has nothing to make ambiguous.
+ subroutine test_c1545_pointer_function_result_agrees
+ integer, pointer :: p
+ integer :: r
+ logical :: flag
+ r = gen_func((flag ? p : func_ptr_result()))
+ end subroutine
+
+ ! Resolution is skipped once C1545 is violated, so the reference gets one
+ ! error rather than a cascade from a resolution that could not succeed.
+ subroutine test_c1545_generic_with_independent_error(flag)
+ logical :: flag
+ integer, allocatable :: a
+ integer :: b
+ !ERROR: In a reference to a generic procedure, if any consequent-arg in a conditional argument has the ALLOCATABLE attribute, each must have it
+ call gen_plain((flag ? a : b), b)
+ end subroutine
+
+ ! A generic intrinsic procedure is a generic procedure, so C1545 applies.
+ subroutine test_f2023_c1545_generic_intrinsic
+ integer, allocatable :: a
+ integer :: b, c, r
+ logical :: flag
+ !ERROR: In a reference to a generic procedure, if any consequent-arg in a conditional argument has the ALLOCATABLE attribute, each must have it
+ r = max((flag ? a : b), c)
+ end subroutine
+
+ subroutine test_c1545_generic_intrinsic_with_independent_error(flag)
+ logical, intent(in) :: flag
+ integer, allocatable :: allocatable_value
+ integer :: plain_value, result
+ !ERROR: In a reference to a generic procedure, if any consequent-arg in a conditional argument has the ALLOCATABLE attribute, each must have it
+ result = max((flag ? allocatable_value : plain_value))
+ end subroutine
+
+ subroutine test_f2023_c1545_generic_intrinsic_subroutine
+ real, allocatable :: allocatable_value
+ real :: plain_value
+ logical :: flag
+ !ERROR: In a reference to a generic procedure, if any consequent-arg in a conditional argument has the ALLOCATABLE attribute, each must have it
+ call random_number((flag ? allocatable_value : plain_value))
+ end subroutine
+
+ ! DSIN is only a specific intrinsic name, so a reference to it is not a
+ ! reference to a generic procedure and C1545 does not apply.
+ subroutine test_c1545_specific_intrinsic
+ double precision, allocatable :: a
+ double precision :: b, r
+ logical :: flag
+ r = dsin((flag ? a : b))
end subroutine
- subroutine test_f2023_c1545_allocatable_consistent_valid
+ subroutine test_f2023_c1545_type_bound_generic
+ type(t_gen) :: obj
+ integer, allocatable :: a
+ integer :: b
+ logical :: flag
+ !ERROR: In a reference to a generic procedure, if any consequent-arg in a conditional argument has the ALLOCATABLE attribute, each must have it
+ call obj%gen((flag ? a : b))
+ end subroutine
+
+ subroutine test_f2023_c1545_generic_allocatable_consistent_valid
integer, allocatable :: a, b
logical :: flag
! Both consequent-args are allocatable -- no C1545 error
- call sub_alloc((flag ? a : b))
+ call gen_plain((flag ? a : b))
end subroutine
- subroutine test_f2023_c1545_pointer_consistent_valid
+ subroutine test_f2023_c1545_generic_pointer_consistent_valid
integer, pointer :: p, q
logical :: flag
! Both consequent-args are pointer -- no C1545 error
- call sub_pointer((flag ? p : q))
+ call gen_plain((flag ? p : q))
+ end subroutine
+
+ ! A reference to a specific procedure is not constrained by C1545, so mixing
+ ! ALLOCATABLE or POINTER consequent-args with plain ones is conforming.
+ subroutine test_c1545_specific_allocatable_mix
+ integer, allocatable :: a
+ integer :: b
+ logical :: flag
+ call sub_int((flag ? a : b))
+ end subroutine
+
+ subroutine test_c1545_specific_pointer_mix
+ integer, pointer :: p
+ integer :: b
+ logical :: flag
+ call sub_int((flag ? p : b))
+ end subroutine
+
+ ! The diagnostics below come from argument association (F2023 15.5.2), not
+ ! from C1545: the dummy argument itself is ALLOCATABLE or POINTER.
+ subroutine test_specific_allocatable_dummy_mismatch
+ integer, allocatable :: a
+ integer :: b
+ logical :: flag
+ !ERROR: ALLOCATABLE dummy argument 'x=' must be associated with an ALLOCATABLE actual argument
+ call sub_alloc((flag ? a : b))
+ end subroutine
+
+ subroutine test_specific_pointer_dummy_mismatch
+ integer, pointer :: p
+ integer :: b
+ logical :: flag
+ !ERROR: Actual argument associated with POINTER dummy argument 'x=' must also be POINTER unless INTENT(IN)
+ call sub_pointer((flag ? p : b))
end subroutine
! =========================================================================
>From 624b49c57d1531e74e41e9c3a0b580f7a91956d3 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron at aaronballman.com>
Date: Tue, 6 Oct 2026 16:09:32 -0400
Subject: [PATCH 14/46] Fix a typo in the release notes; NFC (#229552)
---
clang/docs/ReleaseNotes.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md
index c3ed90f2492aad..25299915441501 100644
--- a/clang/docs/ReleaseNotes.md
+++ b/clang/docs/ReleaseNotes.md
@@ -316,7 +316,7 @@ features cannot lower the translation-unit ABI level;
- Clang now accepts `_single_inheritance` under `-fms-compatibility` as an alias for `__single_inheritance`; `_multiple_inheritance` and `_virtual_inheritance` were already correctly supported as aliases.
- Fixed a bug with handling a `nonnull` attribute with an invalid argument
- index such that it would inadvertantly apply the attribute with no arguments,
+ index such that it would inadvertently apply the attribute with no arguments,
causing all function parameters of pointer type to be considered nonnull. (#GH228670)
### Improvements to Clang's diagnostics
>From c0272181c83e035136dae671f34c95e551044d44 Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer at google.com>
Date: Tue, 6 Oct 2026 13:14:09 -0700
Subject: [PATCH 15/46] [FlowSensitive] add handling for
AssertionResultExpectation (#229273)
In a follow up, this will be used by the StatusOr and Optional models.
Assisted-By: Gemini
---
.../FlowSensitive/Models/CMakeLists.txt | 1 +
.../Models/GtestModelHelpers.cpp | 57 +++++++++++++++++++
.../FlowSensitive/Models/GtestModelHelpers.h | 36 ++++++++++++
3 files changed, 94 insertions(+)
create mode 100644 clang/lib/Analysis/FlowSensitive/Models/GtestModelHelpers.cpp
create mode 100644 clang/lib/Analysis/FlowSensitive/Models/GtestModelHelpers.h
diff --git a/clang/lib/Analysis/FlowSensitive/Models/CMakeLists.txt b/clang/lib/Analysis/FlowSensitive/Models/CMakeLists.txt
index d1236f57148817..62c9473842779d 100644
--- a/clang/lib/Analysis/FlowSensitive/Models/CMakeLists.txt
+++ b/clang/lib/Analysis/FlowSensitive/Models/CMakeLists.txt
@@ -2,6 +2,7 @@ add_clang_library(clangAnalysisFlowSensitiveModels
ChromiumCheckModel.cpp
UncheckedOptionalAccessModel.cpp
UncheckedStatusOrAccessModel.cpp
+ GtestModelHelpers.cpp
LINK_LIBS
clangAnalysis
diff --git a/clang/lib/Analysis/FlowSensitive/Models/GtestModelHelpers.cpp b/clang/lib/Analysis/FlowSensitive/Models/GtestModelHelpers.cpp
new file mode 100644
index 00000000000000..7634ec201f4f84
--- /dev/null
+++ b/clang/lib/Analysis/FlowSensitive/Models/GtestModelHelpers.cpp
@@ -0,0 +1,57 @@
+//===-- GtestModelHelpers.cpp -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines helpers for handling gtest constructs in dataflow models.
+//
+//===----------------------------------------------------------------------===//
+
+#include "GtestModelHelpers.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Analysis/FlowSensitive/StorageLocation.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
+
+using namespace clang::dataflow::gtest;
+using namespace clang::dataflow;
+using namespace clang;
+
+void clang::dataflow::gtest::transferAssertionResultExpectationOperatorBoolCall(
+ const CXXMemberCallExpr *Expr, Environment &Env,
+ llvm::function_ref<StorageLocation &(RecordStorageLocation &)> GetOk) {
+ auto *RecordLoc = getImplicitObjectLocation(*Expr, Env);
+ if (RecordLoc == nullptr)
+ return;
+ RecordStorageLocation *AssertionResultLoc = nullptr;
+ StorageLocation *ExpectedResultLoc = nullptr;
+ for (auto [Field, ChildLoc] : RecordLoc->children()) {
+ if (Field->getName() == "assertion_result")
+ AssertionResultLoc = dyn_cast_or_null<RecordStorageLocation>(ChildLoc);
+ else if (Field->getName() == "expected_result")
+ ExpectedResultLoc = ChildLoc;
+ }
+ if (AssertionResultLoc == nullptr || ExpectedResultLoc == nullptr)
+ return;
+ BoolValue *SuccessVal = Env.get<BoolValue>(GetOk(*AssertionResultLoc));
+ BoolValue *ExpectedVal = Env.get<BoolValue>(*ExpectedResultLoc);
+ if (SuccessVal == nullptr || ExpectedVal == nullptr)
+ return;
+ auto &A = Env.arena();
+ auto &Res = Env.makeAtomicBoolValue();
+ Env.assume(A.makeEquals(Res.formula(), A.makeEquals(SuccessVal->formula(),
+ ExpectedVal->formula())));
+ Env.setValue(*Expr, Res);
+}
+
+clang::ast_matchers::StatementMatcher
+clang::dataflow::gtest::isAssertionResultExpectationOperatorBoolCall() {
+ using namespace clang::ast_matchers;
+ return cxxMemberCallExpr(
+ on(expr(unless(cxxThisExpr()))),
+ callee(cxxMethodDecl(
+ hasName("operator bool"),
+ ofClass(hasName("testing::internal::AssertionResultExpectation")))));
+}
diff --git a/clang/lib/Analysis/FlowSensitive/Models/GtestModelHelpers.h b/clang/lib/Analysis/FlowSensitive/Models/GtestModelHelpers.h
new file mode 100644
index 00000000000000..88009b418222e8
--- /dev/null
+++ b/clang/lib/Analysis/FlowSensitive/Models/GtestModelHelpers.h
@@ -0,0 +1,36 @@
+//===-- GtestModelHelpers.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines helpers for handling gtest constructs in dataflow models.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_ANALYSIS_FLOWSENSITIVE_MODELS_GTESTMODELHELPERS_H
+#define CLANG_ANALYSIS_FLOWSENSITIVE_MODELS_GTESTMODELHELPERS_H
+
+#include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Analysis/FlowSensitive/DataflowEnvironment.h"
+#include "clang/Analysis/FlowSensitive/StorageLocation.h"
+#include <cassert>
+
+namespace clang {
+namespace dataflow {
+namespace gtest {
+void transferAssertionResultExpectationOperatorBoolCall(
+ const CXXMemberCallExpr *Expr, Environment &Env,
+ llvm::function_ref<StorageLocation &(RecordStorageLocation &)> GetOk);
+
+clang::ast_matchers::StatementMatcher
+isAssertionResultExpectationOperatorBoolCall();
+} // namespace gtest
+} // namespace dataflow
+} // namespace clang
+
+#endif // CLANG_ANALYSIS_FLOWSENSITIVE_MODELS_GTESTMODELHELPERS_H
>From d6208a9650bdeb9312d49d2b3e627c6c033d7ec4 Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer at google.com>
Date: Tue, 6 Oct 2026 13:14:19 -0700
Subject: [PATCH 16/46] [FlowSensitive] [Optional] [StatusOr] handle
AssertionResultExpectation (#229274)
gtest changed its implementation of `ASSERT_TRUE` and `ASSERT_FALSE` to
use
`AssertionResultExpectation` in the conditional:
https://github.com/google/googletest/commit/80b3670e1ace250ae2d50fb42ede6646066eb140
Assisted-By: Gemini
---
.../Models/UncheckedOptionalAccessModel.cpp | 9 +++++++++
.../Models/UncheckedStatusOrAccessModel.cpp | 9 +++++++++
.../Analysis/FlowSensitive/MockHeaders.cpp | 18 ++++++++++++++----
3 files changed, 32 insertions(+), 4 deletions(-)
diff --git a/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp b/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
index 568564fb361f4a..063a1a573e6fbe 100644
--- a/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
@@ -35,6 +35,8 @@
#include <cassert>
#include <optional>
+#include "GtestModelHelpers.h"
+
namespace clang {
namespace dataflow {
@@ -1199,6 +1201,13 @@ auto buildTransferMatchSwitch() {
// gtest
.CaseOfCFGStmt<CXXMemberCallExpr>(isAssertionResultOperatorBoolCall(),
transferAssertionResultOperatorBoolCall)
+ .CaseOfCFGStmt<CXXMemberCallExpr>(
+ gtest::isAssertionResultExpectationOperatorBoolCall(),
+ [](const CXXMemberCallExpr *Expr, const MatchFinder::MatchResult &,
+ LatticeTransferState &State) {
+ return gtest::transferAssertionResultExpectationOperatorBoolCall(
+ Expr, State.Env, locForAssertResultSuccess);
+ })
.CaseOfCFGStmt<CXXConstructExpr>(
isAssertionResultConstructFromBoolCall(),
transferAssertionResultConstructFromBoolCall)
diff --git a/clang/lib/Analysis/FlowSensitive/Models/UncheckedStatusOrAccessModel.cpp b/clang/lib/Analysis/FlowSensitive/Models/UncheckedStatusOrAccessModel.cpp
index 6e1bdca22f1c5a..2c8a197ee3b004 100644
--- a/clang/lib/Analysis/FlowSensitive/Models/UncheckedStatusOrAccessModel.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Models/UncheckedStatusOrAccessModel.cpp
@@ -32,6 +32,8 @@
#include "clang/Basic/SourceLocation.h"
#include "llvm/ADT/StringMap.h"
+#include "GtestModelHelpers.h"
+
namespace clang::dataflow::statusor_model {
namespace {
@@ -1241,6 +1243,13 @@ buildTransferMatchSwitch(ASTContext &Ctx,
transferAssertionResultConstructFromBoolCall)
.CaseOfCFGStmt<CXXMemberCallExpr>(isAssertionResultOperatorBoolCall(),
transferAssertionResultOperatorBoolCall)
+ .CaseOfCFGStmt<CXXMemberCallExpr>(
+ gtest::isAssertionResultExpectationOperatorBoolCall(),
+ [](const CXXMemberCallExpr *Expr, const MatchFinder::MatchResult &,
+ LatticeTransferState &State) {
+ return gtest::transferAssertionResultExpectationOperatorBoolCall(
+ Expr, State.Env, locForOk);
+ })
.CaseOfCFGStmt<CXXMemberCallExpr>(isStatusOrMemberCallWithName("ok"),
transferStatusOrOkCall)
.CaseOfCFGStmt<CXXMemberCallExpr>(isStatusOrMemberCallWithName("status"),
diff --git a/clang/unittests/Analysis/FlowSensitive/MockHeaders.cpp b/clang/unittests/Analysis/FlowSensitive/MockHeaders.cpp
index c5ad8ea4dcb1fb..3601d3efbcf1bf 100644
--- a/clang/unittests/Analysis/FlowSensitive/MockHeaders.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/MockHeaders.cpp
@@ -2210,6 +2210,16 @@ GTEST_IMPL_CMP_HELPER_(GT)
#undef GTEST_IMPL_CMP_HELPER_
+struct AssertionResultExpectation {
+ AssertionResult assertion_result;
+ bool expected_result;
+
+ explicit operator bool() const {
+ bool converted(assertion_result);
+ return converted == expected_result;
+ }
+};
+
std::string GetBoolAssertionFailureMessage(
const AssertionResult &assertion_result, const char *expression_text,
const char *actual_predicate_value, const char *expected_predicate_value);
@@ -2334,17 +2344,17 @@ using testing::AssertionResult;
#define GTEST_TEST_BOOLEAN_(expression, text, actual, expected, fail) \
GTEST_AMBIGUOUS_ELSE_BLOCKER_ \
- if (const ::testing::AssertionResult gtest_ar_ = \
- ::testing::AssertionResult(expression)) \
+ if (const ::testing::internal::AssertionResultExpectation gtest_are_ = { \
+ ::testing::AssertionResult(expression), expected}) \
; \
else \
fail(::testing::internal::GetBoolAssertionFailureMessage( \
- gtest_ar_, text, #actual, #expected) \
+ gtest_are_.assertion_result, text, #actual, #expected) \
.c_str())
#define GTEST_ASSERT_TRUE(condition) \
GTEST_TEST_BOOLEAN_(condition, #condition, false, true, GTEST_FATAL_FAILURE_)
#define GTEST_ASSERT_FALSE(condition) \
- GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \
+ GTEST_TEST_BOOLEAN_(condition, #condition, true, false, \
GTEST_FATAL_FAILURE_)
#define ASSERT_TRUE(condition) GTEST_ASSERT_TRUE(condition)
#define ASSERT_FALSE(condition) GTEST_ASSERT_FALSE(condition)
>From d156b688c1b68b3f58b15a7a2d454e610ff50ca8 Mon Sep 17 00:00:00 2001
From: Alex Duran <alejandro.duran at intel.com>
Date: Tue, 6 Oct 2026 22:16:58 +0200
Subject: [PATCH 17/46] [offload][l0] Remove unused DynamicMem structures
(#229513)
The DynamicMemHeapTy and DynamicMemPoolTy structures don't seem to be
used anywhere anymore so this PR cleans them up.
---
.../level_zero/include/L0Memory.h | 30 -------------------
1 file changed, 30 deletions(-)
diff --git a/offload/plugins-nextgen/level_zero/include/L0Memory.h b/offload/plugins-nextgen/level_zero/include/L0Memory.h
index 32bb0a5dc288c1..29a39d4dd77b91 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Memory.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Memory.h
@@ -34,36 +34,6 @@ class L0ContextTy;
constexpr static int32_t MaxMemKind = TARGET_ALLOC_LAST + 1;
-struct DynamicMemHeapTy {
- /// Base address memory is allocated from.
- uintptr_t AllocBase = 0;
- /// Minimal size served by the current heap.
- size_t BlockSize = 0;
- /// Max size served by the current heap.
- size_t MaxSize = 0;
- /// Available memory blocks.
- uint32_t NumBlocks = 0;
- /// Number of block descriptors.
- uint32_t NumBlockDesc = 0;
- /// Number of block counters.
- uint32_t NumBlockCounter = 0;
- /// List of memory block descriptors.
- uint64_t *BlockDesc = nullptr;
- /// List of memory block counters.
- uint32_t *BlockCounter = nullptr;
-};
-
-struct DynamicMemPoolTy {
- /// Location of device memory blocks.
- void *PoolBase = nullptr;
- /// Heap size common to all heaps.
- size_t HeapSize = 0;
- /// Number of heaps available.
- uint32_t NumHeaps = 0;
- /// Heap descriptors (using fixed-size array to simplify memory allocation).
- DynamicMemHeapTy HeapDesc[8];
-};
-
/// Memory allocation information used in memory allocation/deallocation.
struct MemAllocInfoTy {
/// Base address allocated from compute runtime.
>From 78daa43c53d1626e4b2aab409922fda995eb94b5 Mon Sep 17 00:00:00 2001
From: Jason Van Beusekom <jason.van-beusekom at hpe.com>
Date: Tue, 6 Oct 2026 15:19:44 -0500
Subject: [PATCH 18/46] Revert "[flang][mlir][OpenMP] Report Fortran names for
privatized target maps" (#229549)
Reverts llvm/llvm-project#228195, test /target-firstprivate-info.f90 was
failing on certain systems due to a linker error: `undefined symbol:
_FortranAAssignSimple`
---
flang/lib/Lower/Support/Utils.cpp | 6 +-
.../Optimizer/OpenMP/MapInfoFinalization.cpp | 2 +-
.../OpenMP/MapsForPrivatizedSymbols.cpp | 18 +---
.../target-private-allocatable.f90 | 6 +-
.../target-private-implicit-scalar-map-2.f90 | 4 +-
.../target-private-implicit-scalar-map.f90 | 4 +-
...rget-teams-private-implicit-scalar-map.f90 | 6 +-
.../Lower/OpenMP/allocatable-array-bounds.f90 | 6 +-
...allocatable-dtype-intermediate-map-gen.f90 | 6 +-
flang/test/Lower/OpenMP/allocatable-map.f90 | 2 +-
flang/test/Lower/OpenMP/array-bounds.f90 | 2 +-
.../Lower/OpenMP/attach-and-ref-modifier.f90 | 12 +--
flang/test/Lower/OpenMP/declare-mapper.f90 | 20 ++---
flang/test/Lower/OpenMP/defaultmap.f90 | 12 +--
.../OpenMP/derived-type-allocatable-map.f90 | 12 +--
flang/test/Lower/OpenMP/derived-type-map.f90 | 2 +-
...implicit-map-pointer-no-default-mapper.f90 | 4 +-
.../Lower/OpenMP/map-descriptor-deferral.f90 | 18 ++--
.../OpenMP/map-descriptor-privatization.f90 | 2 +-
.../map-neg-alloca-derived-type-array.f90 | 2 +-
.../Lower/OpenMP/optional-argument-map-2.f90 | 6 +-
.../Lower/OpenMP/optional-argument-map-3.f90 | 4 +-
...rget-data-use-device-addr-common-block.f90 | 2 +-
.../target-enter-data-default-openmp52.f90 | 4 +-
.../OpenMP/target-update-derived-type.f90 | 2 +-
flang/test/Lower/OpenMP/target.f90 | 8 +-
.../Lower/OpenMP/usm-descriptor-close-map.f90 | 4 +-
flang/test/Lower/volatile-openmp.f90 | 4 +-
.../Transforms/omp-map-info-finalization.fir | 22 ++---
.../omp-maps-for-privatized-symbols.fir | 8 +-
.../mlir/Target/LLVMIR/Dialect/OpenMPCommon.h | 3 +-
.../OpenMP/OpenMPToLLVMIRTranslation.cpp | 30 ++-----
.../Target/LLVMIR/Dialect/OpenMPCommon.cpp | 17 ++--
.../Target/LLVMIR/omptarget-map-names.mlir | 28 -------
.../fortran/target-firstprivate-info.f90 | 82 -------------------
35 files changed, 112 insertions(+), 258 deletions(-)
delete mode 100644 mlir/test/Target/LLVMIR/omptarget-map-names.mlir
delete mode 100644 offload/test/offloading/fortran/target-firstprivate-info.f90
diff --git a/flang/lib/Lower/Support/Utils.cpp b/flang/lib/Lower/Support/Utils.cpp
index 056e7a5af6431a..e6b6e923308af0 100644
--- a/flang/lib/Lower/Support/Utils.cpp
+++ b/flang/lib/Lower/Support/Utils.cpp
@@ -749,12 +749,8 @@ void privatizeSymbol(
// Boxes should be passed by reference into nested regions:
auto oldIP = firOpBuilder.saveInsertionPoint();
firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
- // Get name so later passes (e.g. MapsForPrivatizedSymbols) can report it in
- // offload info.
- mlir::Location boxLoc = mlir::NameLoc::get(
- firOpBuilder.getStringAttr(sym->name().ToString()), symLoc);
auto alloca =
- fir::AllocaOp::create(firOpBuilder, boxLoc, privVal.getType());
+ fir::AllocaOp::create(firOpBuilder, symLoc, privVal.getType());
firOpBuilder.restoreInsertionPoint(oldIP);
fir::StoreOp::create(firOpBuilder, symLoc, privVal, alloca);
privVal = alloca;
diff --git a/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp b/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
index 6c41425db1d66f..8046a702f330a7 100644
--- a/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
+++ b/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
@@ -590,7 +590,7 @@ class MapInfoFinalizationPass
isRefPtee ? parentOp.getMembersIndexAttr() : mlir::ArrayAttr{},
parentOp.getBounds(),
/*mapperId=*/mapperId,
- /*name=*/parentOp.getNameAttr(),
+ /*name=*/builder.getStringAttr(""),
/*partial_map=*/builder.getBoolAttr(false));
}
diff --git a/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp b/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
index 7ce8ee315df538..de3dd8d352b3b3 100644
--- a/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
+++ b/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
@@ -29,7 +29,6 @@
#include "flang/Optimizer/Dialect/Support/KindMapping.h"
#include "flang/Optimizer/HLFIR/HLFIROps.h"
#include "flang/Optimizer/OpenMP/Passes.h"
-#include "flang/Optimizer/Support/InternalNames.h"
#include "flang/Utils/OpenMP.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
@@ -74,7 +73,6 @@ class MapsForPrivatizedSymbolsPass
Operation *definingOp = var.getDefiningOp();
Value varPtr = var;
- mlir::StringAttr mapName;
// We want the first result of the hlfir.declare op because our goal
// is to map the descriptor (fir.box or fir.boxchar) and the first
// result for hlfir.declare is the descriptor if a the symbol being
@@ -82,19 +80,8 @@ class MapsForPrivatizedSymbolsPass
// Some types are boxed immediately before privatization. These have other
// operations in between the privatization and the declaration. It is safe
// to use var directly here because they will be boxed anyway.
- if (auto declOp = llvm::dyn_cast_if_present<hlfir::DeclareOp>(definingOp)) {
+ if (auto declOp = llvm::dyn_cast_if_present<hlfir::DeclareOp>(definingOp))
varPtr = declOp.getBase();
- std::string sourceName =
- fir::NameUniquer::deconstruct(declOp.getUniqName()).second.name;
- if (!sourceName.empty())
- mapName = builder.getStringAttr(sourceName);
- }
- // Boxed values map an anonymous descriptor with no declare
- // so recover it from the value's location.
- if (!mapName) {
- if (auto nameLoc = llvm::dyn_cast<mlir::NameLoc>(var.getLoc()))
- mapName = nameLoc.getName();
- }
// If we do not have a reference to a descriptor but the descriptor itself,
// then we need to store that on the stack so that we can map the
@@ -169,7 +156,8 @@ class MapsForPrivatizedSymbolsPass
/*members=*/SmallVector<Value>{},
/*member_index=*/mlir::ArrayAttr{},
/*bounds=*/boundsOps,
- /*mapperId=*/mapperId, /*name=*/mapName, builder.getBoolAttr(false));
+ /*mapperId=*/mapperId, /*name=*/StringAttr(),
+ builder.getBoolAttr(false));
}
void addMapInfoOp(omp::TargetOp targetOp, omp::MapInfoOp mapInfoOp) {
auto argIface = llvm::cast<omp::BlockArgOpenMPOpInterface>(*targetOp);
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90
index bafc2622300bbe..8c5ee99b26fd5a 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90
@@ -69,9 +69,9 @@ end subroutine target_allocatable
! CPU-SAME: {bindc_name = "alloc_var", {{.*}}}
! CPU: %[[VAR_DECL:.*]]:2 = hlfir.declare %[[VAR_ALLOC]]
! CPU: %[[BASE_ADDR:.*]] = fir.box_offset %[[VAR_DECL]]#0 base_addr : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> [[MEMBER_TYPE:.*]]
-! CPU: %[[MEMBER:.*]] = omp.map.info var_ptr(%[[VAR_DECL]]#0 : [[TYPE]], [[DESC_TYPE]]) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : [[MEMBER_TYPE:.*]], i32) name("alloc_var") -> {{.*}}
-! CPU: %[[MAP_VAR:.*]] = omp.map.info var_ptr(%[[VAR_DECL]]#0 : [[TYPE]], [[DESC_TYPE]]) map_clauses({{.*}}to{{.*}}) capture(ByRef) members(%[[MEMBER]] : [0] : [[MEMBER_TYPE]]) name("alloc_var") -> !fir.ref<!fir.box<!fir.heap<i32>>>
-! CPU: %[[ATTACH:.*]] = omp.map.info var_ptr(%[[VAR_DECL]]#0 : [[TYPE]], [[DESC_TYPE]]) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : [[MEMBER_TYPE]], i32) name("alloc_var") -> !fir.ref<!fir.box<!fir.heap<i32>>>
+! CPU: %[[MEMBER:.*]] = omp.map.info var_ptr(%[[VAR_DECL]]#0 : [[TYPE]], [[DESC_TYPE]]) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : [[MEMBER_TYPE:.*]], i32) name("") -> {{.*}}
+! CPU: %[[MAP_VAR:.*]] = omp.map.info var_ptr(%[[VAR_DECL]]#0 : [[TYPE]], [[DESC_TYPE]]) map_clauses({{.*}}to{{.*}}) capture(ByRef) members(%[[MEMBER]] : [0] : [[MEMBER_TYPE]]) -> !fir.ref<!fir.box<!fir.heap<i32>>>
+! CPU: %[[ATTACH:.*]] = omp.map.info var_ptr(%[[VAR_DECL]]#0 : [[TYPE]], [[DESC_TYPE]]) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : [[MEMBER_TYPE]], i32) -> !fir.ref<!fir.box<!fir.heap<i32>>>
! CPU: omp.target kernel_type(generic) map_entries(%[[MAP_VAR]] -> %arg0, %[[ATTACH]] -> %arg1, %[[MEMBER]] -> %arg2 : [[TYPE]], [[TYPE]], [[MEMBER_TYPE]]) private(
! CPU-SAME: @[[VAR_PRIVATIZER_SYM]] %[[VAR_DECL]]#0 -> %{{.*}} [map_idx=0] : [[TYPE]]) {
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map-2.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map-2.f90
index fe647147c28e16..dd7bd0a60f8c47 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map-2.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map-2.f90
@@ -35,6 +35,6 @@ end subroutine target_imp_capture
! CHECK-PROG: %[[VAL_1:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.array<10x10xf32>>, !fir.array<10x10xf32>) map_clauses(implicit, tofrom) capture(ByRef) bounds({{.*}}) name("i") -> !fir.ref<!fir.array<10x10xf32>>
! CHECK-PROG: %[[VAL_2:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ptr<!fir.array<5x5x2xf32>>, !fir.array<5x5x2xf32>) map_clauses(implicit, tofrom) capture(ByRef) bounds({{.*}}) name("j") -> !fir.ptr<!fir.array<5x5x2xf32>>
! CHECK-PROG: %[[VAL_3:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ptr<!fir.array<25x2xf32>>, !fir.array<25x2xf32>) map_clauses(implicit, tofrom) capture(ByRef) bounds({{.*}}) name("k") -> !fir.ptr<!fir.array<25x2xf32>>
-! CHECK-PROG: %[[VAL_4:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) name("y") -> !fir.ref<i32>
-! CHECK-PROG: %[[VAL_5:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) name("z") -> !fir.ref<i32>
+! CHECK-PROG: %[[VAL_4:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) -> !fir.ref<i32>
+! CHECK-PROG: %[[VAL_5:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) -> !fir.ref<i32>
! CHECK-PROG: omp.target kernel_type(generic) map_entries(%[[VAL_0]] -> %[[VAL_6:.*]], %[[VAL_1]] -> %[[VAL_7:.*]], %[[VAL_2]] -> %[[VAL_8:.*]], %[[VAL_3]] -> %[[VAL_9:.*]], %[[VAL_4]] -> %[[VAL_10:.*]], %[[VAL_5]] -> %[[VAL_11:.*]] : !fir.ref<i32>, !fir.ref<!fir.array<10x10xf32>>, !fir.ptr<!fir.array<5x5x2xf32>>, !fir.ptr<!fir.array<25x2xf32>>, !fir.ref<i32>, !fir.ref<i32>) private(@_QFtarget_imp_captureEy_firstprivate_i32 %{{.*}}#0 -> %[[VAL_12:.*]] [map_idx=4], @_QMtest_dataEz_firstprivate_i32 %{{.*}}#0 -> %[[VAL_13:.*]] [map_idx=5] : !fir.ref<i32>, !fir.ref<i32>) {
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90
index 5ceeb7f7a03e08..4a8d82ff765ef6 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-implicit-scalar-map.f90
@@ -22,8 +22,8 @@
! CHECK: %[[VAL_1:.*]] = fir.declare %[[VAL_0]] uniq_name("_QFExdgfx") : (!fir.ref<i32>) -> !fir.ref<i32>
! CHECK: %[[VAL_2:.*]] = fir.alloca i32 <{bindc_name = "xfpvx", uniq_name = "_QFExfpvx"}>
! CHECK: %[[VAL_3:.*]] = fir.declare %[[VAL_2]] uniq_name("_QFExfpvx") : (!fir.ref<i32>) -> !fir.ref<i32>
-! CHECK: %[[VAL_4:.*]] = omp.map.info var_ptr(%[[VAL_3]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) name("xfpvx") -> !fir.ref<i32>
-! CHECK: %[[VAL_5:.*]] = omp.map.info var_ptr(%[[VAL_1]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) name("xdgfx") -> !fir.ref<i32>
+! CHECK: %[[VAL_4:.*]] = omp.map.info var_ptr(%[[VAL_3]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) -> !fir.ref<i32>
+! CHECK: %[[VAL_5:.*]] = omp.map.info var_ptr(%[[VAL_1]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) -> !fir.ref<i32>
! CHECK: omp.target kernel_type(generic) map_entries(%[[VAL_4]] -> %{{.*}}, %[[VAL_5]] -> %{{.*}} : !fir.ref<i32>, !fir.ref<i32>) private(@_QFExfpvx_firstprivate_i32 %[[VAL_3]] -> %[[VAL_6:.*]] [map_idx=0], @_QFExdgfx_firstprivate_i32 %[[VAL_1]] -> %[[VAL_7:.*]] [map_idx=1] : !fir.ref<i32>, !fir.ref<i32>) {
! CHECK: %{{.*}} = fir.declare %[[VAL_6]] uniq_name("_QFExfpvx") : (!fir.ref<i32>) -> !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-teams-private-implicit-scalar-map.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-teams-private-implicit-scalar-map.f90
index bb22969113c2b3..aed80b558c3d1d 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-teams-private-implicit-scalar-map.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-teams-private-implicit-scalar-map.f90
@@ -27,11 +27,11 @@ program test_default_implicit_firstprivate
!CHECK: %[[VAL_8:.*]] = omp.map.info var_ptr(%[[VAL_3]] : !fir.ref<i32>, i32) map_clauses(implicit) capture(ByCopy) name("j") -> !fir.ref<i32>
!CHECK: %[[VAL_9:.*]] = omp.map.info var_ptr(%[[VAL_4]] : !fir.ref<i32>, i32) map_clauses(implicit) capture(ByCopy) name("k") -> !fir.ref<i32>
!CHECK: %[[VAL_10:.*]] = fir.box_offset %[[VAL_0]] base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?x?x?xi32>>>
-!CHECK: %[[VAL_11:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>, !fir.box<!fir.heap<!fir.array<?x?x?xi32>>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_10]] : !fir.llvm_ptr<!fir.ref<!fir.array<?x?x?xi32>>>, i32) bounds({{.*}}) name("allocarr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?x?x?xi32>>>
+!CHECK: %[[VAL_11:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>, !fir.box<!fir.heap<!fir.array<?x?x?xi32>>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_10]] : !fir.llvm_ptr<!fir.ref<!fir.array<?x?x?xi32>>>, i32) bounds({{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?x?x?xi32>>>
!CHECK: %[[VAL_12:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>, !fir.box<!fir.heap<!fir.array<?x?x?xi32>>>) map_clauses(always, implicit, to) capture(ByRef) members(%[[VAL_11]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?x?x?xi32>>>) name("allocarr") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>
!CHECK: %[[VAL_13:.*]] = omp.map.info var_ptr(%[[VAL_1]] : !fir.ref<!fir.array<10x10x10xi32>>, !fir.array<10x10x10xi32>) map_clauses(implicit, tofrom) capture(ByRef) bounds({{.*}}) name("arr") -> !fir.ref<!fir.array<10x10x10xi32>>
-!CHECK: %[[VAL_14:.*]] = omp.map.info var_ptr(%[[VAL_6]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) name("xfpvx") -> !fir.ref<i32>
-!CHECK: %[[VAL_15:.*]] = omp.map.info var_ptr(%[[VAL_5]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) name("xdgfx") -> !fir.ref<i32>
+!CHECK: %[[VAL_14:.*]] = omp.map.info var_ptr(%[[VAL_6]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) -> !fir.ref<i32>
+!CHECK: %[[VAL_15:.*]] = omp.map.info var_ptr(%[[VAL_5]] : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) -> !fir.ref<i32>
!CHECK: omp.target kernel_type(spmd) host_eval({{.*}}) map_entries(%[[VAL_7]] -> %{{.*}}, %[[VAL_8]] -> %{{.*}}, %[[VAL_9]] -> %{{.*}}, %[[VAL_12]] -> %{{.*}}, %[[VAL_13]] -> %{{.*}}, %[[VAL_14]] -> %{{.*}}, %[[VAL_15]] -> %{{.*}}, %[[VAL_11]] -> %{{.*}} : {{.*}}) private(@[[SYM_XFPVX]] %[[VAL_6]] -> %{{.*}} [map_idx=5], @[[SYM_XDGFX]] %[[VAL_5]] -> %{{.*}} [map_idx=6] : {{.*}}) {
!CHECK omp.parallel private(@[[SYM_XFPVX]] %{{.*}} -> %{{.*}}, @[[SYM_XDGFX]] %{{.*}} -> %{{.*}}, @[[SYM_I]] %{{.*}} -> %{{.*}}, @[[SYM_J]] %{{.*}} -> %{{.*}}, @[[SYM_K]] %{{.*}} -> %{{.*}} : {{.*}}) {
!$omp target teams distribute parallel do collapse(3) firstprivate(xfpvx)
diff --git a/flang/test/Lower/OpenMP/allocatable-array-bounds.f90 b/flang/test/Lower/OpenMP/allocatable-array-bounds.f90
index 7ffa92f83ac8db..b9d2299a92c0d3 100644
--- a/flang/test/Lower/OpenMP/allocatable-array-bounds.f90
+++ b/flang/test/Lower/OpenMP/allocatable-array-bounds.f90
@@ -23,7 +23,7 @@
!HOST: %[[BOX_3:.*]]:3 = fir.box_dims %[[LOAD_3]], %[[CONSTANT_3]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
!HOST: %[[BOUNDS_1:.*]] = omp.map.bounds lower_bound(%[[LB_1]] : index) upper_bound(%[[UB_1]] : index) extent(%[[BOX_3]]#1 : index) stride(%[[BOX_2]]#2 : index) start_idx(%[[BOX_1]]#0 : index) stride_in_bytes(true)
!HOST: %[[VAR_PTR_PTR:.*]] = fir.box_offset %[[DECLARE_1]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[DECLARE_1]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS_1]]) name("sp_read(2:5)") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[DECLARE_1]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS_1]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!HOST: %[[MAP_INFO_1:.*]] = omp.map.info var_ptr(%[[DECLARE_1]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("sp_read(2:5)") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!HOST: %[[LOAD_3:.*]] = fir.load %[[DECLARE_2]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
@@ -41,7 +41,7 @@
!HOST: %[[BOX_5:.*]]:3 = fir.box_dims %[[LOAD_5]], %[[CONSTANT_5]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
!HOST: %[[BOUNDS_2:.*]] = omp.map.bounds lower_bound(%[[LB_2]] : index) upper_bound(%[[UB_2]] : index) extent(%[[BOX_5]]#1 : index) stride(%[[BOX_4]]#2 : index) start_idx(%[[BOX_3]]#0 : index) stride_in_bytes(true)
!HOST: %[[VAR_PTR_PTR:.*]] = fir.box_offset %[[DECLARE_2]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[DECLARE_2]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS_2]]) name("sp_write(2:5)") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[DECLARE_2]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS_2]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!HOST: %[[MAP_INFO_2:.*]] = omp.map.info var_ptr(%[[DECLARE_2]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("sp_write(2:5)") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
subroutine read_write_section()
@@ -80,7 +80,7 @@ module assumed_allocatable_array_routines
!HOST: %[[BOX_3:.*]]:3 = fir.box_dims %[[LOAD_3]], %[[CONSTANT_3]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
!HOST: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%[[LB]] : index) upper_bound(%[[UB]] : index) extent(%[[BOX_3]]#1 : index) stride(%[[BOX_2]]#2 : index) start_idx(%[[BOX_1]]#0 : index) stride_in_bytes(true)
!HOST: %[[VAR_PTR_PTR:.*]] = fir.box_offset %[[DECLARE]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("arr_read_write(2:5)") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!HOST: %[[MAP_INFO:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("arr_read_write(2:5)") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
subroutine assumed_shape_array(arr_read_write)
integer, allocatable, intent(inout) :: arr_read_write(:)
diff --git a/flang/test/Lower/OpenMP/allocatable-dtype-intermediate-map-gen.f90 b/flang/test/Lower/OpenMP/allocatable-dtype-intermediate-map-gen.f90
index 45c4d840e1be45..789fd1dafa8cb2 100644
--- a/flang/test/Lower/OpenMP/allocatable-dtype-intermediate-map-gen.f90
+++ b/flang/test/Lower/OpenMP/allocatable-dtype-intermediate-map-gen.f90
@@ -9,11 +9,11 @@ subroutine target_map_to()
allocate(derived%scalar)
-!CHECK: %[[SCALAR_DATA:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(to) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("derived%scalar") -> !fir.llvm_ptr<!fir.ref<i32>>
+!CHECK: %[[SCALAR_DATA:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(to) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
!CHECK: %[[SCALAR_DESC:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(always, to) capture(ByRef) name("derived%scalar") -> !fir.ref<!fir.box<!fir.heap<i32>>>
!CHECK: %[[SCALAR_DESC_ATTACH:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("derived%scalar") -> !fir.ref<!fir.box<!fir.heap<i32>>>
-!CHECK: %[[DTYPE_DATA:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>>, !fir.box<!fir.heap<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>) map_clauses(storage) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>, !fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>) name("derived") -> !fir.llvm_ptr<!fir.ref<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>
+!CHECK: %[[DTYPE_DATA:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>>, !fir.box<!fir.heap<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>) map_clauses(storage) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>, !fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>) name("") -> !fir.llvm_ptr<!fir.ref<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>
!CHECK: %[[DTYPE_DESC:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>>, !fir.box<!fir.heap<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>) map_clauses(always, to) capture(ByRef) members({{.*}} : !fir.llvm_ptr<!fir.ref<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>, !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.llvm_ptr<!fir.ref<i32>>) name("derived") -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>>
!CHECK: %[[DTYPE_DESC_ATTACH:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>>, !fir.box<!fir.heap<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>, !fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>) name("derived") -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QFtarget_map_toTdtype{scalar:!fir.box<!fir.heap<i32>>}>>>>
@@ -33,7 +33,7 @@ subroutine update_map_to()
allocate(derived%scalar)
-!CHECK: %[[SCALAR_DATA:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(to) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("derived%scalar") -> !fir.llvm_ptr<!fir.ref<i32>>
+!CHECK: %[[SCALAR_DATA:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(to) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
!CHECK: %[[SCALAR_DESC:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(to) capture(ByRef) members{{.*}} : !fir.llvm_ptr<!fir.ref<i32>>) name("derived%scalar") -> !fir.heap<i32>
!CHECK: %[[SCALAR_DESC_ATTACH:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("derived%scalar") -> !fir.heap<i32>
diff --git a/flang/test/Lower/OpenMP/allocatable-map.f90 b/flang/test/Lower/OpenMP/allocatable-map.f90
index 0b14ceb3fc1ddb..013ba3cf95ead9 100644
--- a/flang/test/Lower/OpenMP/allocatable-map.f90
+++ b/flang/test/Lower/OpenMP/allocatable-map.f90
@@ -2,7 +2,7 @@
!HLFIRDIALECT: %[[POINTER:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QFpointer_routineEpoint") fortran_attrs<pointer> : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
!HLFIRDIALECT: %[[BOX_OFF:.*]] = fir.box_offset %[[POINTER]]#1 base_addr : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
-!HLFIRDIALECT: %[[POINTER_MAP_MEMBER:.*]] = omp.map.info var_ptr(%[[POINTER]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("point") -> !fir.llvm_ptr<!fir.ref<i32>>
+!HLFIRDIALECT: %[[POINTER_MAP_MEMBER:.*]] = omp.map.info var_ptr(%[[POINTER]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
!HLFIRDIALECT: %[[POINTER_MAP:.*]] = omp.map.info var_ptr(%[[POINTER]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(always, to) capture(ByRef) members(%[[POINTER_MAP_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) name("point") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!HLFIRDIALECT: %[[ATTACH_MAP:.*]] = omp.map.info var_ptr(%[[POINTER]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BOX_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("point") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!HLFIRDIALECT: omp.target kernel_type(generic) map_entries(%[[POINTER_MAP]] -> {{.*}}, %[[ATTACH_MAP]] -> {{.*}}, %[[POINTER_MAP_MEMBER]] -> {{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.llvm_ptr<!fir.ref<i32>>) {
diff --git a/flang/test/Lower/OpenMP/array-bounds.f90 b/flang/test/Lower/OpenMP/array-bounds.f90
index d034f166d4d4a0..84ae7e5b1203f4 100644
--- a/flang/test/Lower/OpenMP/array-bounds.f90
+++ b/flang/test/Lower/OpenMP/array-bounds.f90
@@ -51,7 +51,7 @@ module assumed_array_routines
!HOST: %[[DIMS1:.*]]:3 = fir.box_dims %[[ARG0_DECL]]#1, %[[C0_1]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
!HOST: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%[[C3]] : index) upper_bound(%[[C4]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) start_idx(%[[C0]] : index) stride_in_bytes(true)
!HOST: %[[VAR_PTR_PTR:.*]] = fir.box_offset %{{.*}} base_addr : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[INTERMEDIATE_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("arr_read_write(2:5)") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[INTERMEDIATE_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!HOST: %[[MAP:.*]] = omp.map.info var_ptr(%[[INTERMEDIATE_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(target_param, private, attach) capture(ByRef) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) members(%[[MAP_INFO_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("arr_read_write(2:5)") -> !fir.ref<!fir.array<?xi32>>
!HOST: %[[ATTACH_MAP:.*]] = omp.map.info var_ptr(%[[INTERMEDIATE_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("arr_read_write(2:5)") -> !fir.ref<!fir.array<?xi32>>
!HOST: omp.target kernel_type(generic) map_entries(%[[MAP]] -> %{{.*}}, {{.*}} -> {{.*}}, %[[ATTACH_MAP]] -> {{.*}}, %[[MAP_INFO_MEMBER]] -> %{{.*}} : !fir.ref<!fir.array<?xi32>>, !fir.ref<i32>, !fir.ref<!fir.array<?xi32>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) {
diff --git a/flang/test/Lower/OpenMP/attach-and-ref-modifier.f90 b/flang/test/Lower/OpenMP/attach-and-ref-modifier.f90
index 701c389ad84d3c..cd13b3d66e0870 100644
--- a/flang/test/Lower/OpenMP/attach-and-ref-modifier.f90
+++ b/flang/test/Lower/OpenMP/attach-and-ref-modifier.f90
@@ -6,7 +6,7 @@ subroutine attach_always()
!CHECK: func.func @{{.*}}attach_always{{.*}}
!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %{{.*}} {{.*}} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
!CHECK: %[[BASE_ADDR:.*]] = fir.box_offset %[[DECLARE]]#1 base_addr : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
-!CHECK: %[[MAP_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("x") -> !fir.llvm_ptr<!fir.ref<i32>>
+!CHECK: %[[MAP_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
!CHECK: %[[MAP_DESCRIPTOR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(always, to) capture(ByRef) members(%[[MAP_BASE_ADDR]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) name("x") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK: %[[MAP_ATTACH:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(always, attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("x") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK: omp.target kernel_type(generic) map_entries(%[[MAP_DESCRIPTOR]] -> %{{.*}}, %[[MAP_ATTACH]] -> %{{.*}}, %[[MAP_BASE_ADDR]] -> %{{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.llvm_ptr<!fir.ref<i32>>) {
@@ -21,7 +21,7 @@ subroutine attach_never()
!CHECK: func.func @{{.*}}attach_never{{.*}}
!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %{{.*}} {{.*}} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
!CHECK: %[[BASE_ADDR:.*]] = fir.box_offset %[[DECLARE]]#1 base_addr : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
-!CHECK: %[[MAP_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("x") -> !fir.llvm_ptr<!fir.ref<i32>>
+!CHECK: %[[MAP_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
!CHECK: %[[MAP_DESCRIPTOR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(always, to) capture(ByRef) members(%[[MAP_BASE_ADDR]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) name("x") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK-NOT: %[[MAP_ATTACH:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(always, attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("x") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK: omp.target kernel_type(generic) map_entries(%[[MAP_DESCRIPTOR]] -> %{{.*}}, %[[MAP_BASE_ADDR]] -> %{{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.llvm_ptr<!fir.ref<i32>>) {
@@ -35,7 +35,7 @@ subroutine attach_auto()
!CHECK: func.func @{{.*}}attach_auto{{.*}}
!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %{{.*}} {{.*}} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
!CHECK: %[[BASE_ADDR:.*]] = fir.box_offset %[[DECLARE]]#1 base_addr : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
-!CHECK: %[[MAP_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("x") -> !fir.llvm_ptr<!fir.ref<i32>>
+!CHECK: %[[MAP_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
!CHECK: %[[MAP_DESCRIPTOR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(always, to) capture(ByRef) members(%[[MAP_BASE_ADDR]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) name("x") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK: %[[MAP_ATTACH:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("x") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK: omp.target kernel_type(generic) map_entries(%[[MAP_DESCRIPTOR]] -> %{{.*}}, %[[MAP_ATTACH]] -> %{{.*}}, %[[MAP_BASE_ADDR]] -> %{{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.llvm_ptr<!fir.ref<i32>>) {
@@ -50,7 +50,7 @@ subroutine ref_ptr_ptee()
!CHECK: func.func @{{.*}}ref_ptr_ptee{{.*}}
!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %{{.*}} {{.*}} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
!CHECK: %[[BASE_ADDR:.*]] = fir.box_offset %[[DECLARE]]#1 base_addr : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
-!CHECK: %[[MAP_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(to, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("x") -> !fir.llvm_ptr<!fir.ref<i32>>
+!CHECK: %[[MAP_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(to, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
!CHECK: %[[MAP_DESCRIPTOR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(to, ref_ptr, ref_ptee) capture(ByRef) members(%[[MAP_BASE_ADDR]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) name("x") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK: %[[MAP_ATTACH:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("x") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK: omp.target kernel_type(generic) map_entries(%[[MAP_DESCRIPTOR]] -> %{{.*}}, %[[MAP_ATTACH]] -> %{{.*}}, %[[MAP_BASE_ADDR]] -> %{{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.llvm_ptr<!fir.ref<i32>>) {
@@ -81,7 +81,7 @@ subroutine ref_ptee()
!CHECK: func.func @{{.*}}ref_ptee{{.*}}
!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %{{.*}} {{.*}} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
!CHECK: %[[BASE_ADDR:.*]] = fir.box_offset %[[DECLARE]]#1 base_addr : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
-!CHECK: %[[MAP_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(to, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("x") -> !fir.llvm_ptr<!fir.ref<i32>>
+!CHECK: %[[MAP_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(to, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
!CHECK-NOT: %[[MAP_DESCRIPTOR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses({{.*}}) capture(ByRef) members(%[[MAP_BASE_ADDR]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) name("x") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK: %[[MAP_ATTACH:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(attach, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("x") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK: omp.target kernel_type(generic) map_entries(%[[MAP_BASE_ADDR]] -> %{{.*}}, %[[MAP_ATTACH]] -> %{{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, !fir.ref<!fir.box<!fir.ptr<i32>>>) {
@@ -96,7 +96,7 @@ subroutine ref_ptr_ptee_attach_never()
!CHECK: func.func @{{.*}}ref_ptr_ptee_attach_never{{.*}}
!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %{{.*}} {{.*}} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
!CHECK: %[[BASE_ADDR_1:.*]] = fir.box_offset %[[DECLARE]]#1 base_addr : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
-!CHECK: %[[MAP_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(to, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR_1]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("x") -> !fir.llvm_ptr<!fir.ref<i32>>
+!CHECK: %[[MAP_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(to, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR_1]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
!CHECK: %[[MAP_DESCRIPTOR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(to, ref_ptr, ref_ptee) capture(ByRef) members(%[[MAP_BASE_ADDR]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) name("x") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK-NOT: %[[BASE_ADDR_2:.*]] = fir.box_offset %[[DECLARE]]#1 base_addr : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
!CHECK-NOT: %[[MAP_ATTACH:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses({{.*}}) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR_2]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("x") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
diff --git a/flang/test/Lower/OpenMP/declare-mapper.f90 b/flang/test/Lower/OpenMP/declare-mapper.f90
index a2e963e27415a4..bd9d604d9dd025 100644
--- a/flang/test/Lower/OpenMP/declare-mapper.f90
+++ b/flang/test/Lower/OpenMP/declare-mapper.f90
@@ -51,7 +51,7 @@ subroutine declare_mapper_1
!CHECK: %[[VAL_16:.*]] = omp.map.bounds lower_bound(%[[VAL_10]] : index) upper_bound(%[[VAL_15]] : index) extent(%[[VAL_6]]#1 : index) stride(%[[VAL_8]] : index) start_idx(%[[VAL_6]]#0 : index)
!CHECK: %[[VAL_18:.*]] = fir.coordinate_of %[[VAL_1]]#0, values : (!fir.ref<[[MY_TYPE]]>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[VAL_19:.*]] = fir.box_offset %[[VAL_18]] base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
- !CHECK: %[[VAL_20:.*]] = omp.map.info var_ptr(%[[VAL_18]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_19]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[VAL_16]]) name("var%values(1:var%num_vals)") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+ !CHECK: %[[VAL_20:.*]] = omp.map.info var_ptr(%[[VAL_18]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_19]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[VAL_16]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[VAL_21:.*]] = omp.map.info var_ptr(%[[VAL_18]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) name("var%[[VAL_22:.*]](1:var%[[VAL_23:.*]])") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[VAL_ATTACH:.*]] = omp.map.info var_ptr(%[[VAL_18]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[VAL_16]]) name("var%values(1:var%num_vals)") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[VAL_24:.*]] = omp.map.info var_ptr(%[[VAL_1]]#1 : !fir.ref<[[MY_TYPE]]>, [[MY_TYPE]]) map_clauses(tofrom) capture(ByRef) members(%[[VAL_21]], %[[VAL_20]] : [1], [1, 0] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("var") -> !fir.ref<[[MY_TYPE]]>
@@ -144,7 +144,7 @@ subroutine declare_mapper_3
!CHECK: %[[VAL_16:.*]] = omp.map.bounds lower_bound(%[[VAL_10]] : index) upper_bound(%[[VAL_15]] : index) extent(%[[VAL_6]]#1 : index) stride(%[[VAL_8]] : index) start_idx(%[[VAL_6]]#0 : index)
!CHECK: %[[VAL_18:.*]] = fir.coordinate_of %[[VAL_1]]#0, values : (!fir.ref<[[MY_TYPE]]>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[VAL_19:.*]] = fir.box_offset %[[VAL_18]] base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
- !CHECK: %[[VAL_20:.*]] = omp.map.info var_ptr(%[[VAL_18]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_19]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[VAL_16]]) name("var%values(1:var%num_vals)") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+ !CHECK: %[[VAL_20:.*]] = omp.map.info var_ptr(%[[VAL_18]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_19]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[VAL_16]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[VAL_21:.*]] = omp.map.info var_ptr(%[[VAL_18]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) name("var%[[VAL_22:.*]](1:var%[[VAL_23:.*]])") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[ATTACH_MAP:.*]] = omp.map.info var_ptr(%[[VAL_18]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("var%values(1:var%num_vals)") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[VAL_24:.*]] = omp.map.info var_ptr(%[[VAL_1]]#1 : !fir.ref<[[MY_TYPE]]>, [[MY_TYPE]]) map_clauses(tofrom) capture(ByRef) members(%[[VAL_21]], %[[VAL_20]] : [1], [1, 0] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("var") -> !fir.ref<[[MY_TYPE]]>
@@ -382,13 +382,13 @@ subroutine declare_mapper_10
integer :: var_a, var_b
! dtype (dtype_a) should have mapper applied via var_ptr_ptr
- ! CHECK: omp.map.info {{.*}} mapper(@[[MAPPER_A]]){{.*}}name("dtype")
+ ! CHECK: omp.map.info {{.*}} mapper(@[[MAPPER_A]]){{.*}}name("")
! CHECK: omp.map.info {{.*}} name("dtype")
! var_a and var_b are integers - no mapper
! CHECK: omp.map.info {{.*}} map_clauses(to) capture(ByRef) name("var_a") -> !fir.ref<i32>
! CHECK: omp.map.info {{.*}} map_clauses(to) capture(ByRef) name("var_b") -> !fir.ref<i32>
! dtype2 (dtype_b) should NOT have mapper applied
- ! CHECK: omp.map.info {{.*}} map_clauses(to) capture(ByRef) var_ptr_ptr({{.*}}) name("dtype2") -> {{.*}}
+ ! CHECK: omp.map.info {{.*}} map_clauses(to) capture(ByRef) var_ptr_ptr({{.*}}) name("") -> {{.*}}
! CHECK-NOT: mapper(@
! CHECK: omp.map.info {{.*}} name("dtype2")
! CHECK: omp.target_enter_data
@@ -418,13 +418,13 @@ subroutine declare_mapper_11
integer :: var_a, var_b
! dtype (dtype_a) should use named mapper "testing" when explicitly specified
- ! CHECK: omp.map.info {{.*}} mapper(@[[MAPPER_TESTING]]){{.*}}name("dtype")
+ ! CHECK: omp.map.info {{.*}} mapper(@[[MAPPER_TESTING]]){{.*}}name("")
! CHECK: omp.map.info {{.*}} name("dtype")
! var_a and var_b are integers - no mapper
! CHECK: omp.map.info {{.*}} map_clauses(to) capture(ByRef) name("var_a") -> !fir.ref<i32>
! CHECK: omp.map.info {{.*}} map_clauses(to) capture(ByRef) name("var_b") -> !fir.ref<i32>
! dtype2 (dtype_b) should NOT have mapper - no mapper defined for dtype_b
- ! CHECK: omp.map.info {{.*}} map_clauses(to) capture(ByRef) var_ptr_ptr({{.*}}) name("dtype2") -> {{.*}}
+ ! CHECK: omp.map.info {{.*}} map_clauses(to) capture(ByRef) var_ptr_ptr({{.*}}) name("") -> {{.*}}
! CHECK-NOT: mapper(@
! CHECK: omp.map.info {{.*}} name("dtype2")
! CHECK: omp.target_enter_data
@@ -471,16 +471,16 @@ subroutine declare_mapper_12
! - dtype4 (dtype_d): no mapper (no mapper defined for dtype_d)
! dtype should get MAPPER_A (default for dtype_a)
- ! CHECK: omp.map.info {{.*}} mapper(@[[MAPPER_A]]){{.*}}name("dtype")
+ ! CHECK: omp.map.info {{.*}} mapper(@[[MAPPER_A]]){{.*}}name("")
! CHECK: omp.map.info {{.*}} name("dtype")
! dtype2 should get MAPPER_B (default for dtype_b)
- ! CHECK: omp.map.info {{.*}} mapper(@[[MAPPER_B]]){{.*}}name("dtype2")
+ ! CHECK: omp.map.info {{.*}} mapper(@[[MAPPER_B]]){{.*}}name("")
! CHECK: omp.map.info {{.*}} name("dtype2")
! dtype3 should get MAPPER_TESTING (explicit match)
- ! CHECK: omp.map.info {{.*}} mapper(@[[MAPPER_TESTING]]){{.*}}name("dtype3")
+ ! CHECK: omp.map.info {{.*}} mapper(@[[MAPPER_TESTING]]){{.*}}name("")
! CHECK: omp.map.info {{.*}} name("dtype3")
! dtype4 should NOT have any mapper
- ! CHECK: omp.map.info {{.*}} map_clauses(to) capture(ByRef) var_ptr_ptr({{.*}}) name("dtype4") -> {{.*}}
+ ! CHECK: omp.map.info {{.*}} map_clauses(to) capture(ByRef) var_ptr_ptr({{.*}}) name("") -> {{.*}}
! CHECK-NOT: mapper(@
! CHECK: omp.map.info {{.*}} name("dtype4")
! CHECK: omp.target_enter_data
diff --git a/flang/test/Lower/OpenMP/defaultmap.f90 b/flang/test/Lower/OpenMP/defaultmap.f90
index 438f292109c44f..6dca29434c34d8 100644
--- a/flang/test/Lower/OpenMP/defaultmap.f90
+++ b/flang/test/Lower/OpenMP/defaultmap.f90
@@ -8,7 +8,7 @@ subroutine defaultmap_allocatable_present()
implicit none
integer, dimension(:), allocatable :: arr
-! CHECK: %[[MAP_1:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(implicit, present) capture(ByRef) var_ptr_ptr({{.*}}) bounds({{.*}}) name("arr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+! CHECK: %[[MAP_1:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(implicit, present) capture(ByRef) var_ptr_ptr({{.*}}) bounds({{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
! CHECK: %[[MAP_2:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, implicit, to) capture(ByRef) members({{.*}}) name("arr") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!$omp target defaultmap(present: allocatable)
arr(1) = 10
@@ -36,7 +36,7 @@ subroutine defaultmap_all_default()
integer :: scalar_int
! CHECK: %[[MAP_1:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<i32>, i32) map_clauses(implicit) capture(ByCopy) name("scalar_int") -> !fir.ref<i32>
-! CHECK: %[[MAP_2:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr({{.*}}) bounds({{.*}}) name("arr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+! CHECK: %[[MAP_2:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr({{.*}}) bounds({{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
! CHECK: %[[MAP_3:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, implicit, to) capture(ByRef) members({{.*}}) name("arr") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
! CHECK: %[[MAP_4:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.array<16xi32>>, !fir.array<16xi32>) map_clauses(implicit, tofrom) capture(ByRef) bounds({{.*}}) name("aggregate") -> !fir.ref<!fir.array<16xi32>>
@@ -53,10 +53,10 @@ subroutine defaultmap_pointer_to()
integer, dimension(:), pointer :: arr_ptr(:)
integer :: scalar_int
-! CHECK-NO-FPRIV: %[[MAP_1:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(implicit, to) capture(ByRef) var_ptr_ptr({{.*}}) bounds({{.*}}) name("arr_ptr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-! CHECK-FPRIV: %[[MAP_1:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(implicit, to) capture(ByRef) var_ptr_ptr({{.*}}) bounds({{.*}}) name("arr_ptr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+! CHECK-NO-FPRIV: %[[MAP_1:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(implicit, to) capture(ByRef) var_ptr_ptr({{.*}}) bounds({{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+! CHECK-FPRIV: %[[MAP_1:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(implicit, to) capture(ByRef) var_ptr_ptr({{.*}}) bounds({{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
! CHECK: %[[MAP_2:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(always, implicit, to) capture(ByRef) members({{.*}}) name("arr_ptr") -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
-! CHECK-FPRIV: %[[MAP_3:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) name("scalar_int") -> !fir.ref<i32>
+! CHECK-FPRIV: %[[MAP_3:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) -> !fir.ref<i32>
! CHECK-NO-FPRIV: %[[MAP_3:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<i32>, i32) map_clauses(implicit) capture(ByCopy) name("scalar_int") -> !fir.ref<i32>
!$omp target defaultmap(to: pointer)
arr_ptr(1) = scalar_int + 20
@@ -120,7 +120,7 @@ subroutine defaultmap_scalar_implicit_mapper()
type(dtype), allocatable :: obj
! CHECK-LABEL: func.func @_QPdefaultmap_scalar_implicit_mapper
-! CHECK: %[[BASE_MAP:.*]] = omp.map.info {{.*}} map_clauses(implicit, tofrom) capture(ByRef){{.*}} name("obj") -> {{.*}}
+! CHECK: %[[BASE_MAP:.*]] = omp.map.info {{.*}} map_clauses(implicit, tofrom) capture(ByRef){{.*}} name("") -> {{.*}}
! CHECK: %[[DESC_MAP:.*]] = omp.map.info {{.*}} map_clauses(always, implicit, to) capture(ByRef) members(%[[BASE_MAP]] : [0] : {{.*}}) name("obj") -> {{.*}}
! CHECK: omp.target kernel_type(generic) map_entries(%[[DESC_MAP]] -> {{.*}}, %[[BASE_MAP]] -> {{.*}})
allocate(obj)
diff --git a/flang/test/Lower/OpenMP/derived-type-allocatable-map.f90 b/flang/test/Lower/OpenMP/derived-type-allocatable-map.f90
index b9959e04d29f4e..736da39c257e47 100644
--- a/flang/test/Lower/OpenMP/derived-type-allocatable-map.f90
+++ b/flang/test/Lower/OpenMP/derived-type-allocatable-map.f90
@@ -5,7 +5,7 @@
!CHECK: %[[BOUNDS:.*]] = omp.map.bounds lower_bound({{.*}}) upper_bound({{.*}}) extent({{.*}}) stride({{.*}}) start_idx({{.*}}) stride_in_bytes(true)
!CHECK: %[[MEMBER_COORD:.*]] = fir.coordinate_of %[[DECLARE]]#0, array_j : (!fir.ref<!fir.type<[[ONE_LAYER_TY]]>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[MEMBER_BASE_ADDR:.*]] = fir.box_offset %[[MEMBER_COORD]] base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[MAP_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("one_l%array_j") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[MAP_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[MAP_MEMBER_DESCRIPTOR:.*]] = omp.map.info var_ptr(%[[MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) name("one_l%array_j") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[MAP_MEMBER_ATTACH:.*]] = omp.map.info var_ptr(%[[MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds({{.*}}) name("one_l%array_j") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[MAP_PARENT:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.type<[[ONE_LAYER_TY]]>>, !fir.type<[[ONE_LAYER_TY]]>) map_clauses(storage) capture(ByRef) members(%[[MAP_MEMBER_DESCRIPTOR]], %[[MAP_MEMBER_BASE_ADDR]] : [4], [4, 0] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("one_l") partial_map(true) -> !fir.ref<!fir.type<[[ONE_LAYER_TY]]>>
@@ -36,14 +36,14 @@ subroutine dtype_alloca_map_op_block()
!CHECK: %[[LOAD_DTYPE:.*]] = fir.load %[[DECLARE]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>
!CHECK: %[[MEMBER_COORD:.*]] = fir.coordinate_of %[[LOAD_DTYPE]], array_j : (!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[MEMBER_BASE_ADDR:.*]] = fir.box_offset %[[MEMBER_COORD]] base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[MAP_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("one_l%array_j") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[MAP_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[MAP_MEMBER_DESC:.*]] = omp.map.info var_ptr(%[[MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) name("one_l%array_j") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[MAP_MEMBER_ATTACH:.*]] = omp.map.info var_ptr(%[[MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds({{.*}}) name("one_l%array_j") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[LOAD_DTYPE:.*]] = fir.load %[[DECLARE]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>
!CHECK: %[[REGULAR_MEMBER:.*]] = fir.coordinate_of %[[LOAD_DTYPE]], k : (!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) -> !fir.ref<i32>
!CHECK: %[[MAP_REGULAR_MEMBER:.*]] = omp.map.info var_ptr(%[[REGULAR_MEMBER]] : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) name("one_l%k") -> !fir.ref<i32>
!CHECK: %[[DTYPE_BASE_ADDR:.*]] = fir.box_offset %[[DECLARE]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>
-!CHECK: %[[MAP_DTYPE_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) map_clauses(storage) capture(ByRef) var_ptr_ptr(%[[DTYPE_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.type<[[REC_TY]]>) name("one_l") -> !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>
+!CHECK: %[[MAP_DTYPE_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) map_clauses(storage) capture(ByRef) var_ptr_ptr(%[[DTYPE_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.type<[[REC_TY]]>) name("") -> !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>
!CHECK: %[[MAP_DTYPE_DESC:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) map_clauses(always, to) capture(ByRef) members(%[[MAP_DTYPE_BASE_ADDR]], %[[MAP_MEMBER_DESC]], %[[MAP_MEMBER_BASE_ADDR]], %[[MAP_REGULAR_MEMBER]] : [0], [0, 4], [0, 4, 0], [0, 5] : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, !fir.ref<i32>) name("one_l") -> !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>
!CHECK: %[[MAP_DTYPE_DESC_ATTACH:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[DTYPE_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.type<[[REC_TY]]>) name("one_l") -> !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>
!CHECK: omp.target kernel_type(generic) map_entries(%[[MAP_DTYPE_DESC]] -> %[[ARG0:.*]], %[[MAP_MEMBER_DESC]] -> %[[ARG1:.*]], %[[MAP_REGULAR_MEMBER]] -> %[[ARG2:.*]], %[[MAP_MEMBER_ATTACH]] -> %[[ARG3:.*]], %[[MAP_DTYPE_DESC_ATTACH]] -> %[[ARG4:.*]], %[[MAP_DTYPE_BASE_ADDR]] -> %[[ARG5:.*]], %[[MAP_MEMBER_BASE_ADDR]] -> %[[ARG6:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<i32>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) {
@@ -76,7 +76,7 @@ subroutine alloca_dtype_op_block_add()
!CHECK: %[[NESTED_DTYPE_COORD:.*]] = fir.coordinate_of %[[LOAD]], nest : (!fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>) -> !fir.ref<!fir.type<[[REC_TY2:_QFalloca_nest_dype_map_op_block_addTmiddle_layer{i:f32,array_i:!fir.array<10xi32>,array_k:!fir.box<!fir.heap<!fir.array<\?xi32>>>,k:i32}]]>>
!CHECK: %[[NESTED_MEMBER_COORD:.*]] = fir.coordinate_of %[[NESTED_DTYPE_COORD]], array_k : (!fir.ref<!fir.type<[[REC_TY2]]>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[NESTED_MEMBER_BASE_ADDR:.*]] = fir.box_offset %[[NESTED_MEMBER_COORD]] base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[MAP_NESTED_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[NESTED_MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[NESTED_MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("one_l%nest%array_k") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[MAP_NESTED_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[NESTED_MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[NESTED_MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[MAP_NESTED_MEMBER_COORD:.*]] = omp.map.info var_ptr(%[[NESTED_MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) name("one_l%nest%array_k") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[MAP_NESTED_MEMBER_ATTACH:.*]] = omp.map.info var_ptr(%[[NESTED_MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds({{.*}}) name("one_l%nest%array_k") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[LOAD:.*]] = fir.load %[[DECLARE]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>>
@@ -84,7 +84,7 @@ subroutine alloca_dtype_op_block_add()
!CHECK: %[[REGULAR_NESTED_MEMBER_COORD:.*]] = fir.coordinate_of %[[NESTED_DTYPE_COORD]], k : (!fir.ref<!fir.type<[[REC_TY2]]>>) -> !fir.ref<i32>
!CHECK: %[[MAP_REGULAR_NESTED_MEMBER:.*]] = omp.map.info var_ptr(%[[REGULAR_NESTED_MEMBER_COORD]] : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) name("one_l%nest%k") -> !fir.ref<i32>
!CHECK: %[[DTYPE_BASE_ADDR:.*]] = fir.box_offset %[[DECLARE]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>}>>>
-!CHECK: %[[MAP_DTYPE_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>) map_clauses(storage) capture(ByRef) var_ptr_ptr(%[[DTYPE_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>}>>>, !fir.type<[[REC_TY]]>}>) name("one_l") -> !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>}>>>
+!CHECK: %[[MAP_DTYPE_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>) map_clauses(storage) capture(ByRef) var_ptr_ptr(%[[DTYPE_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>}>>>, !fir.type<[[REC_TY]]>}>) name("") -> !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>}>>>
!CHECK: %[[MAP_DTYPE:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>) map_clauses(always, to) capture(ByRef) members(%[[MAP_DTYPE_BASE_ADDR]], %[[MAP_NESTED_MEMBER_COORD]], %[[MAP_NESTED_MEMBER_BASE_ADDR]], %[[MAP_REGULAR_NESTED_MEMBER]] : [0], [0, 6, 2], [0, 6, 2, 0], [0, 6, 3] : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>}>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, !fir.ref<i32>) name("one_l") -> !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>>
!CHECK: %[[MAP_DTYPE_ATTACH:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>}>>>, !fir.type<[[REC_TY]]>}>) {{.*}}
!CHECK: omp.target kernel_type(generic) map_entries(%[[MAP_DTYPE]] -> %[[ARG0:.*]], %[[MAP_NESTED_MEMBER_COORD]] -> %[[ARG1:.*]], %[[MAP_REGULAR_NESTED_MEMBER]] -> %[[ARG2:.*]], %[[MAP_NESTED_MEMBER_ATTACH]] -> %[[ARG3:.*]], %[[MAP_DTYPE_ATTACH]] -> %[[ARG4:.*]], %[[MAP_DTYPE_BASE_ADDR]] -> %[[ARG5:.*]], %[[MAP_NESTED_MEMBER_BASE_ADDR]] -> %[[ARG6:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<i32>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>}>>>>, !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>}>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) {
@@ -124,7 +124,7 @@ subroutine alloca_nest_dype_map_op_block_add()
!CHECK: %[[NESTED_DTYPE_COORD:.*]] = fir.coordinate_of %[[DECLARE]]#0, nest : (!fir.ref<!fir.type<[[REC_TY]]>>) -> !fir.ref<!fir.type<[[REC_TY2:_QFnest_dtype_alloca_map_op_block_addTmiddle_layer{i:f32,array_i:!fir.array<10xi32>,array_k:!fir.box<!fir.heap<!fir.array<\?xi32>>>,k:i32}]]>>
!CHECK: %[[NESTED_MEMBER_COORD:.*]] = fir.coordinate_of %[[NESTED_DTYPE_COORD]], array_k : (!fir.ref<!fir.type<[[REC_TY2]]>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[NESTED_MEMBER_BASE_ADDR:.*]] = fir.box_offset %[[NESTED_MEMBER_COORD]] base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[MAP_NESTED_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[NESTED_MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[NESTED_MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("one_l%nest%array_k") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[MAP_NESTED_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[NESTED_MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[NESTED_MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[MAP_NESTED_MEMBER_DESC:.*]] = omp.map.info var_ptr(%[[NESTED_MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) name("one_l%nest%array_k") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[MAP_NESTED_MEMBER_DESC_ATTACH:.*]] = omp.map.info var_ptr(%[[NESTED_MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds({{.*}}) name("one_l%nest%array_k") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[MAP_PARENT:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.type<[[REC_TY]]>>, !fir.type<[[REC_TY]]>) map_clauses(storage) capture(ByRef) members(%[[MAP_NESTED_MEMBER_DESC]], %[[MAP_NESTED_MEMBER_BASE_ADDR]] : [6, 2], [6, 2, 0] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("one_l") partial_map(true) -> !fir.ref<!fir.type<[[REC_TY]]>>
diff --git a/flang/test/Lower/OpenMP/derived-type-map.f90 b/flang/test/Lower/OpenMP/derived-type-map.f90
index 936c498766e98a..7b0e4b2cc61e4a 100644
--- a/flang/test/Lower/OpenMP/derived-type-map.f90
+++ b/flang/test/Lower/OpenMP/derived-type-map.f90
@@ -22,7 +22,7 @@ end subroutine mapType_derived_implicit
!CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>> <{bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_implicit_allocatableEscalar_arr"}>
!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] uniq_name("_QFmaptype_derived_implicit_allocatableEscalar_arr") fortran_attrs<allocatable> : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>, !fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>)
!CHECK: %[[BOX_ADDR:.*]] = fir.box_offset %[[DECLARE]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>
-!CHECK: %[[BASE_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>, !fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>, !fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) name("scalar_arr") -> !fir.llvm_ptr<!fir.ref<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>
+!CHECK: %[[BASE_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>, !fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>, !fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) name("") -> !fir.llvm_ptr<!fir.ref<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>
!CHECK: %[[DESC_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>, !fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>) map_clauses(always, implicit, to) capture(ByRef) members(%[[BASE_MAP]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>) name("scalar_arr") -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>
!CHECK: %[[ATTACH_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>, !fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>, !fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) name("scalar_arr") -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>
!CHECK: omp.target kernel_type(generic) map_entries(%[[DESC_MAP]] -> %[[ARG0:.*]], %[[ATTACH_MAP]] -> %[[ARG1:.*]], %[[BASE_MAP]] -> %[[ARG2:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>, !fir.ref<!fir.box<!fir.heap<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>>, !fir.llvm_ptr<!fir.ref<!fir.type<_QFmaptype_derived_implicit_allocatableTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>>) {
diff --git a/flang/test/Lower/OpenMP/implicit-map-pointer-no-default-mapper.f90 b/flang/test/Lower/OpenMP/implicit-map-pointer-no-default-mapper.f90
index 087a91f199b004..fd270c6f669fa8 100644
--- a/flang/test/Lower/OpenMP/implicit-map-pointer-no-default-mapper.f90
+++ b/flang/test/Lower/OpenMP/implicit-map-pointer-no-default-mapper.f90
@@ -20,9 +20,9 @@ program p
! CHECK-LABEL: func.func @_QQmain
! The pointer capture should not get an implicit default mapper.
-! CHECK: %[[PTR_PTEE_MAP:.*]] = omp.map.info {{.*}}map_clauses(implicit, tofrom){{.*}} name("ptr")
+! CHECK: %[[PTR_PTEE_MAP:.*]] = omp.map.info {{.*}}map_clauses(implicit, tofrom){{.*}} name("")
! CHECK: %[[PTR_DESC_MAP:.*]] = omp.map.info {{.*}}members(%[[PTR_PTEE_MAP]]{{.*}}){{.*}} name("ptr")
! The allocatable capture should still use the implicit default mapper.
-! CHECK: %[[ALLOC_PTEE_MAP:.*]] = omp.map.info {{.*}}map_clauses(implicit, tofrom){{.*}}mapper(@_QQFt_omp_default_mapper){{.*}} name("al")
+! CHECK: %[[ALLOC_PTEE_MAP:.*]] = omp.map.info {{.*}}map_clauses(implicit, tofrom){{.*}}mapper(@_QQFt_omp_default_mapper){{.*}} name("")
! CHECK: %[[ALLOC_DESC_MAP:.*]] = omp.map.info {{.*}}members(%[[ALLOC_PTEE_MAP]]{{.*}}){{.*}} name("al")
diff --git a/flang/test/Lower/OpenMP/map-descriptor-deferral.f90 b/flang/test/Lower/OpenMP/map-descriptor-deferral.f90
index ae2cbc09862c9d..72e084a29dbeae 100644
--- a/flang/test/Lower/OpenMP/map-descriptor-deferral.f90
+++ b/flang/test/Lower/OpenMP/map-descriptor-deferral.f90
@@ -22,7 +22,7 @@ subroutine assume_map_target_enter_exit(assumed_arr)
!CHECK: %[[MAP_ADDR:.*]] = omp.map.info var_ptr(%[[LOAD_BOX]] : !fir.ref<!fir.array<?xi32>>, i32) map_clauses(to) capture(ByRef) bounds(%{{.*}}) name("assumed_arr") -> !fir.ref<!fir.array<?xi32>>
!CHECK: omp.target_enter_data map_entries(%[[MAP_ADDR]] : !fir.ref<!fir.array<?xi32>>)
!CHECK: %[[BOX_ADDR:.*]] = fir.box_offset %{{.*}} base_addr : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[MAP_ADDR:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[MAP_ADDR:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[MAP_BOX:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(implicit, target_param, private, attach) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) members(%{{.*}} : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("assumed_arr") -> !fir.ref<!fir.array<?xi32>>
!CHECK: %[[MAP_BOX_ATTACH:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.ref<!fir.array<?xi32>>
!CHECK: omp.target kernel_type(generic) map_entries(%[[MAP_BOX]] -> %{{.*}}, %[[MAP_BOX_ATTACH]] -> %{{.*}}, %[[MAP_ADDR]] -> %{{.*}} : !fir.ref<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) {
@@ -42,17 +42,17 @@ subroutine assume_alloca_map_target_enter_exit(assumed_arr)
!CHECK-LABEL: func.func @_QPassume_alloca_map_target_enter_exit(
!CHECK: %[[BOX_ADDR:.*]] = fir.box_offset %{{.*}} base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[BOX_ADDR_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(to) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[BOX_ADDR_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(to) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[DESC_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) members(%[[BOX_ADDR_MAP]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("assumed_arr") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[ATTACH_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: omp.target_enter_data map_entries(%[[DESC_MAP]], %[[ATTACH_MAP]], %[[BOX_ADDR_MAP]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>)
!CHECK: %[[BOX_ADDR:.*]] = fir.box_offset %{{.*}} base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[BOX_ADDR_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[BOX_ADDR_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[DESC_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, implicit, to) capture(ByRef) members(%[[BOX_ADDR_MAP]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("assumed_arr") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[ATTACH_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: omp.target kernel_type(generic) map_entries(%[[DESC_MAP]] -> %[[VAL_28:.*]], %[[ATTACH_MAP]] -> %[[VAL_29:.*]], %[[BOX_ADDR_MAP]] -> %[[VAL_30:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) {
!CHECK: %[[BOX_ADDR:.*]] = fir.box_offset %{{.*}} base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[BOX_ADDR_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(from) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[BOX_ADDR_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(from) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[DESC_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(from) capture(ByRef) members(%[[BOX_ADDR_MAP]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("assumed_arr") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: %[[ATTACH_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
!CHECK: omp.target_exit_data map_entries(%[[DESC_MAP]], %[[ATTACH_MAP]], %[[BOX_ADDR_MAP]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>)
@@ -68,17 +68,17 @@ subroutine assume_pointer_map_target_enter_exit(assumed_arr)
!CHECK-LABEL: func.func @_QPassume_pointer_map_target_enter_exit(
!CHECK: %[[BOX_ADDR:.*]] = fir.box_offset %{{.*}} base_addr : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[BOX_ADDR_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(to) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[BOX_ADDR_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(to) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[DESC_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) members(%[[BOX_ADDR_MAP]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("assumed_arr") -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
!CHECK: %[[ATTACH_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
!CHECK: omp.target_enter_data map_entries(%[[DESC_MAP]], %[[ATTACH_MAP]], %[[BOX_ADDR_MAP]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>)
!CHECK: %[[BOX_ADDR:.*]] = fir.box_offset %{{.*}} base_addr : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[BOX_ADDR_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[BOX_ADDR_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[DESC_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(always, implicit, to) capture(ByRef) members(%[[BOX_ADDR_MAP]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("assumed_arr") -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
!CHECK: %[[ATTACH_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
!CHECK: omp.target kernel_type(generic) map_entries(%[[DESC_MAP]] -> %[[VAL_28:.*]], %[[ATTACH_MAP]] -> %[[VAL_29:.*]], %[[BOX_ADDR_MAP]] -> %[[VAL_30:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) {
!CHECK: %[[BOX_ADDR:.*]] = fir.box_offset %{{.*}} base_addr : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[BOX_ADDR_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(from) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[BOX_ADDR_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(from) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[DESC_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(from) capture(ByRef) members(%[[BOX_ADDR_MAP]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("assumed_arr") -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
!CHECK: %[[ATTACH_MAP:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
!CHECK: omp.target_exit_data map_entries(%[[DESC_MAP]], %[[ATTACH_MAP]], %[[BOX_ADDR_MAP]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>)
@@ -94,12 +94,12 @@ subroutine assume_map_target_data(assumed_arr)
!CHECK-LABEL: func.func @_QPassume_map_target_data(
!CHECK: %[[BOX_ADDR:.*]] = fir.box_offset %{{.*}} base_addr : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[MAP_ADDR:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(to) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[MAP_ADDR:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(to) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[MAP_BOX:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(always, to) capture(ByRef) members(%[[MAP_ADDR]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("assumed_arr") -> !fir.ref<!fir.array<?xi32>>
!CHECK: %[[ATTACH:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.ref<!fir.array<?xi32>>
!CHECK: omp.target_data map_entries(%[[MAP_BOX]], %[[ATTACH]], %[[MAP_ADDR]] : !fir.ref<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) {
!CHECK: %[[BOX_ADDR:.*]] = fir.box_offset %{{.*}} base_addr : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[MAP_ADDR:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[MAP_ADDR:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[MAP_BOX:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(implicit, target_param, private, attach) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) members(%[[MAP_ADDR]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("assumed_arr") -> !fir.ref<!fir.array<?xi32>>
!CHECK: %[[ATTACH:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("assumed_arr") -> !fir.ref<!fir.array<?xi32>>
!CHECK: omp.target kernel_type(generic) map_entries(%[[MAP_BOX]] -> %{{.*}}, %[[ATTACH]] -> %{{.*}}, %[[MAP_ADDR]] -> %{{.*}} : !fir.ref<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) {
diff --git a/flang/test/Lower/OpenMP/map-descriptor-privatization.f90 b/flang/test/Lower/OpenMP/map-descriptor-privatization.f90
index 63c82600ef624c..8c5f0dd8a18900 100644
--- a/flang/test/Lower/OpenMP/map-descriptor-privatization.f90
+++ b/flang/test/Lower/OpenMP/map-descriptor-privatization.f90
@@ -13,7 +13,7 @@ subroutine assumed_shape_array_priv(arr_read_write)
!CHECK-LABEL: func.func @_QPassumed_shape_array_priv(
!CHECK: %[[DESC_ALLOCA:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
!CHECK: %[[BOX_ADDR:.*]] = fir.box_offset %[[DESC_ALLOCA]] base_addr : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-!CHECK: %[[MAP_MEMBER:.*]] = omp.map.info var_ptr(%[[DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("arr_read_write") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!CHECK: %[[MAP_MEMBER:.*]] = omp.map.info var_ptr(%[[DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
!CHECK: %[[MAP_PARENT:.*]] = omp.map.info var_ptr(%[[DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(target_param, private, attach) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) members(%[[MAP_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("arr_read_write") -> !fir.ref<!fir.array<?xi32>>
!CHECK: %[[MAP_ATTACH:.*]] = omp.map.info var_ptr(%[[DESC_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BOX_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("arr_read_write") -> !fir.ref<!fir.array<?xi32>>
!CHECK: omp.target kernel_type(generic) map_entries(%[[MAP_PARENT]] -> %{{.*}}, %[[MAP_ATTACH]] -> %{{.*}}, %[[MAP_MEMBER]] -> %{{.*}} : !fir.ref<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) {
diff --git a/flang/test/Lower/OpenMP/map-neg-alloca-derived-type-array.f90 b/flang/test/Lower/OpenMP/map-neg-alloca-derived-type-array.f90
index a0b8388dd256bb..c5d7b3864f2a26 100644
--- a/flang/test/Lower/OpenMP/map-neg-alloca-derived-type-array.f90
+++ b/flang/test/Lower/OpenMP/map-neg-alloca-derived-type-array.f90
@@ -23,5 +23,5 @@ subroutine map_negative_bounds_allocatable_dtype()
! CHECK: %[[VAL_10:.*]] = fir.array_coor %{{.*}}(%[[VAL_9]]) %[[VAL_2]], %[[VAL_4]] : (!fir.heap<!fir.array<?x?x!fir.type<_QFmap_negative_bounds_allocatable_dtypeTderived_type{data:!fir.box<!fir.ptr<!fir.array<?x?x?xf32>>>}>>>, !fir.shapeshift<2>, index, index) -> !fir.ref<!fir.type<_QFmap_negative_bounds_allocatable_dtypeTderived_type{data:!fir.box<!fir.ptr<!fir.array<?x?x?xf32>>>}>>
! CHECK: %[[VAL_11:.*]] = fir.coordinate_of %[[VAL_10]], data : (!fir.ref<!fir.type<_QFmap_negative_bounds_allocatable_dtypeTderived_type{data:!fir.box<!fir.ptr<!fir.array<?x?x?xf32>>>}>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xf32>>>>
! CHECK: %[[VAL_12:.*]] = fir.box_offset %[[VAL_11]] base_addr : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xf32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?x?x?xf32>>>
-! CHECK: %[[VAL_13:.*]] = omp.map.info var_ptr(%[[VAL_11]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xf32>>>>, !fir.box<!fir.ptr<!fir.array<?x?x?xf32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_12]] : !fir.llvm_ptr<!fir.ref<!fir.array<?x?x?xf32>>>, f32) bounds({{.*}}) name("dtype(-1_8,1_8)%data") -> !fir.llvm_ptr<!fir.ref<!fir.array<?x?x?xf32>>>
+! CHECK: %[[VAL_13:.*]] = omp.map.info var_ptr(%[[VAL_11]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xf32>>>>, !fir.box<!fir.ptr<!fir.array<?x?x?xf32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_12]] : !fir.llvm_ptr<!fir.ref<!fir.array<?x?x?xf32>>>, f32) bounds({{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?x?x?xf32>>>
! CHECK: %[[VAL_14:.*]] = omp.map.info var_ptr(%[[VAL_11]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xf32>>>>, !fir.box<!fir.ptr<!fir.array<?x?x?xf32>>>) map_clauses(always, to) capture(ByRef) name({{.*}}) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xf32>>>>
diff --git a/flang/test/Lower/OpenMP/optional-argument-map-2.f90 b/flang/test/Lower/OpenMP/optional-argument-map-2.f90
index b0e8bdacbedf9d..9aac46632c0694 100644
--- a/flang/test/Lower/OpenMP/optional-argument-map-2.f90
+++ b/flang/test/Lower/OpenMP/optional-argument-map-2.f90
@@ -72,9 +72,9 @@ end module mod
! CHECK-FPRIV: %[[VAL_13:.*]] = arith.subi %[[VAL_12]]#1, %[[VAL_11]] : index
! CHECK-FPRIV: %[[VAL_14:.*]] = omp.map.bounds lower_bound(%[[VAL_10]] : index) upper_bound(%[[VAL_13]] : index) extent(%[[VAL_12]]#1 : index) stride(%[[VAL_11]] : index) start_idx(%[[VAL_10]] : index) stride_in_bytes(true)
! CHECK-FPRIV: %[[VAL_16:.*]] = fir.box_offset %[[VAL_0]] base_addr : (!fir.ref<!fir.boxchar<1>>) -> !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>
-! CHECK-FPRIV: %[[VAL_17:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_16]] : !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>, !fir.char<1,?>) bounds(%[[VAL_14]]) name("a") -> !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>
-! CHECK-FPRIV: %[[VAL_18:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(always, to) capture(ByRef) members(%[[VAL_17]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>) name("a") -> !fir.ref<!fir.boxchar<1>>
-! CHECK-FPRIV: %[[VAL_20:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[VAL_16]] : !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>, !fir.char<1,?>) bounds(%[[VAL_14]]) name("a") -> !fir.ref<!fir.boxchar<1>>
+! CHECK-FPRIV: %[[VAL_17:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_16]] : !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>, !fir.char<1,?>) bounds(%[[VAL_14]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>
+! CHECK-FPRIV: %[[VAL_18:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(always, to) capture(ByRef) members(%[[VAL_17]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>) -> !fir.ref<!fir.boxchar<1>>
+! CHECK-FPRIV: %[[VAL_20:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[VAL_16]] : !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>, !fir.char<1,?>) bounds(%[[VAL_14]]) -> !fir.ref<!fir.boxchar<1>>
! CHECK-FPRIV: omp.target kernel_type(generic) map_entries(%[[VAL_7]] -> %[[VAL_21:.*]], %[[VAL_18]] -> %[[VAL_22:.*]], [[VAL_20:.*]] -> %{{.*}}, %[[VAL_17]] -> %[[VAL_23:.*]] : !fir.ref<!fir.char<1,4>>, !fir.ref<!fir.boxchar<1>>, !fir.ref<!fir.boxchar<1>>, !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>) private(@_QMmodFroutine_boxcharEa_firstprivate_boxchar_c8xU %[[VAL_3]]#0 -> %[[VAL_24:.*]] [map_idx=1] : !fir.boxchar<1>) {
! CHECK-FPRIV: %[[VAL_25:.*]] = arith.constant 4 : index
! CHECK-FPRIV: %[[VAL_26:.*]]:2 = hlfir.declare %[[VAL_21]] typeparams %[[VAL_25]] uniq_name("_QMmodFroutine_boxcharEb") : (!fir.ref<!fir.char<1,4>>, index) -> (!fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,4>>)
diff --git a/flang/test/Lower/OpenMP/optional-argument-map-3.f90 b/flang/test/Lower/OpenMP/optional-argument-map-3.f90
index d0669265b3d70b..e0b762f615d06f 100644
--- a/flang/test/Lower/OpenMP/optional-argument-map-3.f90
+++ b/flang/test/Lower/OpenMP/optional-argument-map-3.f90
@@ -32,7 +32,7 @@ end subroutine foo
! CHECK: fir.store %[[VAL_1]]#1 to %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
! CHECK: }
! CHECK: %[[VAL_3:.*]] = fir.box_offset %[[VAL_0]] base_addr : (!fir.ref<!fir.box<!fir.array<?xf32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>
-! CHECK: %[[VAL_4:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.box<!fir.array<?xf32>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_3]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>, f32) bounds(%{{.*}}) name("dt") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>
+! CHECK: %[[VAL_4:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.box<!fir.array<?xf32>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_3]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>, f32) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>
! CHECK: %[[VAL_5:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.box<!fir.array<?xf32>>) map_clauses(implicit, target_param, private, attach) capture(ByRef) var_ptr_ptr(%[[VAL_3]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>, f32) members(%[[VAL_4]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>) name("dt") -> !fir.ref<!fir.array<?xf32>>
! CHECK: omp.target kernel_type(spmd) host_eval({{.*}}) map_entries({{.*}}%[[VAL_5]] -> {{.*}}, %[[VAL_4]] -> {{.*}} : {{.*}}) {
! CHECK: } else {
@@ -41,6 +41,6 @@ end subroutine foo
! CHECK: fir.store %[[VAL_1]]#1 to %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xf32>>>
! CHECK: }
! CHECK: %[[VAL_7:.*]] = fir.box_offset %[[VAL_0]] base_addr : (!fir.ref<!fir.box<!fir.array<?xf32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>
-! CHECK: %[[VAL_8:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.box<!fir.array<?xf32>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_7]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>, f32) bounds(%{{.*}}) name("dt") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>
+! CHECK: %[[VAL_8:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.box<!fir.array<?xf32>>) map_clauses(implicit, tofrom) capture(ByRef) var_ptr_ptr(%[[VAL_7]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>, f32) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>
! CHECK: %[[VAL_9:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.box<!fir.array<?xf32>>) map_clauses(implicit, target_param, private, attach) capture(ByRef) var_ptr_ptr(%[[VAL_7]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>, f32) members(%[[VAL_8]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xf32>>>) name("dt") -> !fir.ref<!fir.array<?xf32>>
! CHECK: omp.target kernel_type(spmd) host_eval({{.*}}) map_entries({{.*}}, %[[VAL_9]] ->{{.*}}, %[[VAL_8]] -> {{.*}} : {{.*}}) {
diff --git a/flang/test/Lower/OpenMP/target-data-use-device-addr-common-block.f90 b/flang/test/Lower/OpenMP/target-data-use-device-addr-common-block.f90
index 770e363ef7d35e..2d18437f97cbf3 100644
--- a/flang/test/Lower/OpenMP/target-data-use-device-addr-common-block.f90
+++ b/flang/test/Lower/OpenMP/target-data-use-device-addr-common-block.f90
@@ -153,7 +153,7 @@ subroutine outer_map_common
! CHECK-LABEL: func.func @_QPreordered_common
! CHECK: %[[SECOND_R_MAP:.*]] = omp.map.info {{.*}} map_clauses(return_param) {{.*}} {{(\{name = "r"\}|name\("r"\))}}
! CHECK: %[[SECOND_S_MAP:.*]] = omp.map.info {{.*}} map_clauses(return_param) {{.*}} {{(\{name = "s"\}|name\("s"\))}}
-! CHECK: %[[PTR_CHILD:.*]] = omp.map.info {{.*}} map_clauses(return_param) {{.*}} {{(\{name = "ptr"\}|name\("ptr"\))}}
+! CHECK: %[[PTR_CHILD:.*]] = omp.map.info {{.*}} map_clauses(return_param) {{.*}} {{(\{name = ""\}|name\(""\))}}
! CHECK: %[[PTR_MAP:.*]] = omp.map.info {{.*}} members(%[[PTR_CHILD]] {{.*}}) {{.*}}{{(\{name = "ptr"\}|name\("ptr"\))}}
! CHECK: %[[PTR_ATTACH:.*]] = omp.map.info {{.*}} map_clauses(attach, ref_ptr, ref_ptee) {{.*}} {{(\{name = "ptr"\}|name\("ptr"\))}}
! CHECK: %[[FIRST_A_MAP:.*]] = omp.map.info {{.*}} map_clauses(return_param) {{.*}} {{(\{name = "a"\}|name\("a"\))}}
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
index ab553167118f5c..086866cb8682bf 100644
--- a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -10,7 +10,7 @@ module test
subroutine initialize()
allocate(A)
!$omp target enter data map(A)
- !CHECK-52: omp.map.info var_ptr(%2 : !fir.ref<!fir.box<!fir.heap<f32>>>, !fir.box<!fir.heap<f32>>) map_clauses(to) capture(ByRef) var_ptr_ptr(%5 : !fir.llvm_ptr<!fir.ref<f32>>, f32) name("a") -> !fir.llvm_ptr<!fir.ref<f32>>
+ !CHECK-52: omp.map.info var_ptr(%2 : !fir.ref<!fir.box<!fir.heap<f32>>>, !fir.box<!fir.heap<f32>>) map_clauses(to) capture(ByRef) var_ptr_ptr(%5 : !fir.llvm_ptr<!fir.ref<f32>>, f32) name("") -> !fir.llvm_ptr<!fir.ref<f32>>
!CHECK-52: omp.map.info var_ptr(%2 : !fir.ref<!fir.box<!fir.heap<f32>>>, !fir.box<!fir.heap<f32>>) map_clauses(always, to) capture(ByRef) members(%6 : [0] : !fir.llvm_ptr<!fir.ref<f32>>) name("a") -> !fir.ref<!fir.box<!fir.heap<f32>>>
!CHECK-51: to and alloc map types are permitted
@@ -18,7 +18,7 @@ end subroutine initialize
subroutine finalize()
!$omp target exit data map(A)
- !CHECK-52: omp.map.info var_ptr(%2 : !fir.ref<!fir.box<!fir.heap<f32>>>, !fir.box<!fir.heap<f32>>) map_clauses(from) capture(ByRef) var_ptr_ptr(%3 : !fir.llvm_ptr<!fir.ref<f32>>, f32) name("a") -> !fir.llvm_ptr<!fir.ref<f32>>
+ !CHECK-52: omp.map.info var_ptr(%2 : !fir.ref<!fir.box<!fir.heap<f32>>>, !fir.box<!fir.heap<f32>>) map_clauses(from) capture(ByRef) var_ptr_ptr(%3 : !fir.llvm_ptr<!fir.ref<f32>>, f32) name("") -> !fir.llvm_ptr<!fir.ref<f32>>
!CHECK-52: omp.map.info var_ptr(%2 : !fir.ref<!fir.box<!fir.heap<f32>>>, !fir.box<!fir.heap<f32>>) map_clauses(from) capture(ByRef) members(%4 : [0] : !fir.llvm_ptr<!fir.ref<f32>>) name("a") -> !fir.ref<!fir.box<!fir.heap<f32>>>
!CHECK-51: from, release and delete map types are permitted
deallocate(A)
diff --git a/flang/test/Lower/OpenMP/target-update-derived-type.f90 b/flang/test/Lower/OpenMP/target-update-derived-type.f90
index 343c157bb08cb0..aefc78398e774a 100644
--- a/flang/test/Lower/OpenMP/target-update-derived-type.f90
+++ b/flang/test/Lower/OpenMP/target-update-derived-type.f90
@@ -118,7 +118,7 @@ subroutine update_pointer(w)
type(wavefun) :: w
! CHECK: %[[FERWE_MAP:.*]] = omp.map.info {{.*}} map_clauses(to)
- ! CHECK: %[[PTR_MAP:.*]] = omp.map.info {{.*}} map_clauses(to) {{.*}}members({{.*}}name("w%ptr")
+ ! CHECK: %[[PTR_MAP:.*]] = omp.map.info {{.*}} map_clauses(to) {{.*}}name("w%ptr")
! CHECK: omp.target_update map_entries(%[[FERWE_MAP]], %[[PTR_MAP]],
!$omp target update to(w%ferwe, w%ptr)
end subroutine
diff --git a/flang/test/Lower/OpenMP/target.f90 b/flang/test/Lower/OpenMP/target.f90
index 45379e370813cf..47a9c8bc8b8546 100644
--- a/flang/test/Lower/OpenMP/target.f90
+++ b/flang/test/Lower/OpenMP/target.f90
@@ -548,10 +548,10 @@ subroutine omp_target_device_addr
integer, pointer :: a
!CHECK: %[[VAL_0:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> <{bindc_name = "a", uniq_name = "_QFomp_target_device_addrEa"}>
!CHECK: %[[VAL_0_DECL:.*]]:2 = hlfir.declare %[[VAL_0]] uniq_name("_QFomp_target_device_addrEa") fortran_attrs<pointer> : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
- !CHECK: %[[MAP_MEMBERS:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("a") -> !fir.llvm_ptr<!fir.ref<i32>>
+ !CHECK: %[[MAP_MEMBERS:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
!CHECK: %[[MAP:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(always, to) capture(ByRef) members(%[[MAP_MEMBERS]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) name("a") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK: %[[MAP_ATTACH:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("a") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
- !CHECK: %[[DEV_ADDR_MEMBERS:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(return_param) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("a") -> !fir.llvm_ptr<!fir.ref<i32>>
+ !CHECK: %[[DEV_ADDR_MEMBERS:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(return_param) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
!CHECK: %[[DEV_ADDR:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(always, to) capture(ByRef) members(%[[DEV_ADDR_MEMBERS]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) name("a") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK: %[[DEV_ADDR_ATTACH:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("a") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
!CHECK: omp.target_data map_entries(%[[MAP]], %[[MAP_ATTACH]], %[[DEV_ADDR_ATTACH]], %[[MAP_MEMBERS]] : {{.*}}) use_device_addr(%[[DEV_ADDR]] -> %[[ARG_0:.*]], %[[DEV_ADDR_MEMBERS]] -> %[[ARG_1:.*]] : {{.*}}) {
@@ -677,8 +677,8 @@ subroutine target_unstructured
!CHECK-NO-FPRIV: %[[VAL_4:.*]] = omp.map.info var_ptr(%[[VAL_1]]#1 : !fir.ref<i32>, i32) map_clauses(implicit) capture(ByCopy) name("i") -> !fir.ref<i32>
!CHECK-NO-FPRIV: %[[VAL_5:.*]] = omp.map.info var_ptr(%[[VAL_3]]#1 : !fir.ref<i32>, i32) map_clauses(implicit) capture(ByCopy) name("j") -> !fir.ref<i32>
!CHECK-NO-FPRIV: omp.target kernel_type(generic) map_entries(%[[VAL_4]] -> %[[VAL_6:.*]], %[[VAL_5]] -> %[[VAL_7:.*]] : !fir.ref<i32>, !fir.ref<i32>) {
- !CHECK-FPRIV: %[[VAL_4:.*]] = omp.map.info var_ptr(%[[VAL_1]]#0 : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) name("i") -> !fir.ref<i32>
- !CHECK-FPRIV: %[[VAL_5:.*]] = omp.map.info var_ptr(%[[VAL_3]]#0 : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) name("j") -> !fir.ref<i32>
+ !CHECK-FPRIV: %[[VAL_4:.*]] = omp.map.info var_ptr(%[[VAL_1]]#0 : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) -> !fir.ref<i32>
+ !CHECK-FPRIV: %[[VAL_5:.*]] = omp.map.info var_ptr(%[[VAL_3]]#0 : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) -> !fir.ref<i32>
!CHECK-FPRIV: omp.target kernel_type(generic) map_entries(%[[VAL_4]] -> %[[ARG_0:.*]], %[[VAL_5]] -> %[[ARG_1:.*]] : !fir.ref<i32>, !fir.ref<i32>) private(@{{.*}} %[[VAL_1]]#0 -> %[[ARG_2:.*]] [map_idx=0], @{{.*}} %[[VAL_3]]#0 -> %[[ARG_3:.*]] [map_idx=1] : !fir.ref<i32>, !fir.ref<i32>) {
!$omp target
!CHECK-FPRIV: %[[VAL_8:.*]]:2 = hlfir.declare %[[ARG_2]] uniq_name("_QFtarget_unstructuredEi") : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenMP/usm-descriptor-close-map.f90 b/flang/test/Lower/OpenMP/usm-descriptor-close-map.f90
index a8c664976d6005..2438295510820b 100644
--- a/flang/test/Lower/OpenMP/usm-descriptor-close-map.f90
+++ b/flang/test/Lower/OpenMP/usm-descriptor-close-map.f90
@@ -11,7 +11,7 @@ module usm_descriptor_close_map
! CHECK-LABEL: func.func @_QMusm_descriptor_close_mapPusm_only
! CHECK: %[[DECL:.*]]:2 = hlfir.declare %{{.*}} {{.*}}uniq_name({{.*}}Epoint") fortran_attrs<pointer>
! CHECK: %[[BOX_OFF:.*]] = fir.box_offset %[[DECL]]#1 base_addr : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
-! CHECK: %[[MEMBER:.*]] = omp.map.info var_ptr(%[[DECL]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("point") -> !fir.llvm_ptr<!fir.ref<i32>>
+! CHECK: %[[MEMBER:.*]] = omp.map.info var_ptr(%[[DECL]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
! CHECK: %[[DESC_MAP:.*]] = omp.map.info var_ptr(%[[DECL]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(always, to) capture(ByRef) members(%[[MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) name("point") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
subroutine usm_only(point)
integer, pointer :: point
@@ -23,7 +23,7 @@ subroutine usm_only(point)
! CHECK-LABEL: func.func @_QMusm_descriptor_close_mapPusm_explicit_close
! CHECK: %[[DECL:.*]]:2 = hlfir.declare %{{.*}} {{.*}}uniq_name({{.*}}Epoint") fortran_attrs<pointer>
! CHECK: %[[BOX_OFF:.*]] = fir.box_offset %[[DECL]]#1 base_addr : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
-! CHECK: %[[MEMBER:.*]] = omp.map.info var_ptr(%[[DECL]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(close, tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("point") -> !fir.llvm_ptr<!fir.ref<i32>>
+! CHECK: %[[MEMBER:.*]] = omp.map.info var_ptr(%[[DECL]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(close, tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
! CHECK: %[[DESC_MAP:.*]] = omp.map.info var_ptr(%[[DECL]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(close, to) capture(ByRef) members(%[[MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) name("point") -> !fir.ref<!fir.box<!fir.ptr<i32>>>
subroutine usm_explicit_close(point)
integer, pointer :: point
diff --git a/flang/test/Lower/volatile-openmp.f90 b/flang/test/Lower/volatile-openmp.f90
index 297d8ea1ee27a8..3448be0bd6bd4d 100644
--- a/flang/test/Lower/volatile-openmp.f90
+++ b/flang/test/Lower/volatile-openmp.f90
@@ -35,7 +35,7 @@
! CHECK: %[[VAL_23:.*]] = omp.map.bounds lower_bound(%[[VAL_0]] : index) upper_bound(%[[VAL_22]] : index) extent(%[[VAL_21]]#1 : index) stride(%[[VAL_21]]#2 : index) start_idx(%[[VAL_21]]#0 : index) stride_in_bytes(true)
! CHECK: %[[VAL_24:.*]] = fir.coordinate_of %[[VAL_13]]#0, array : (!fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
! CHECK: %[[VAL_25:.*]] = fir.box_offset %[[VAL_24]] base_addr : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-! CHECK: %[[VAL_26:.*]] = omp.map.info var_ptr(%[[VAL_24]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(to) capture(ByRef) var_ptr_ptr(%[[VAL_25]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[VAL_23]]) name("container%array") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+! CHECK: %[[VAL_26:.*]] = omp.map.info var_ptr(%[[VAL_24]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(to) capture(ByRef) var_ptr_ptr(%[[VAL_25]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[VAL_23]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
! CHECK: %[[VAL_27:.*]] = omp.map.info var_ptr(%[[VAL_24]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) name("container%[[VAL_28:.*]]") -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
! CHECK: %[[VAL_ATTACH_1:.*]] = omp.map.info var_ptr(%[[VAL_24]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.box<!fir.ptr<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds({{.*}}) name("container%array") -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
! CHECK: %[[VAL_29:.*]] = omp.map.info var_ptr(%[[VAL_13]]#1 : !fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>, !fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>) map_clauses(storage) capture(ByRef) members(%[[VAL_27]], %[[VAL_26]] : [0], [0, 0] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("container") partial_map(true) -> !fir.ref<!fir.type<_QFTt{array:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>, volatile>
@@ -47,7 +47,7 @@
! CHECK: %[[VAL_34:.*]] = arith.subi %[[VAL_33]]#1, %[[VAL_1]] : index
! CHECK: %[[VAL_35:.*]] = omp.map.bounds lower_bound(%[[VAL_0]] : index) upper_bound(%[[VAL_34]] : index) extent(%[[VAL_33]]#1 : index) stride(%[[VAL_33]]#2 : index) start_idx(%[[VAL_32]]#0 : index) stride_in_bytes(true)
! CHECK: %[[VAL_36:.*]] = fir.box_offset %[[VAL_10]]#1 base_addr : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>, volatile>, volatile>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-! CHECK: %[[VAL_37:.*]] = omp.map.info var_ptr(%[[VAL_10]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>, volatile>, volatile>, !fir.box<!fir.ptr<!fir.array<?xi32>>, volatile>) map_clauses(to) capture(ByRef) var_ptr_ptr(%[[VAL_36]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[VAL_35]]) name("array1") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+! CHECK: %[[VAL_37:.*]] = omp.map.info var_ptr(%[[VAL_10]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>, volatile>, volatile>, !fir.box<!fir.ptr<!fir.array<?xi32>>, volatile>) map_clauses(to) capture(ByRef) var_ptr_ptr(%[[VAL_36]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[VAL_35]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
! CHECK: %[[VAL_38:.*]] = omp.map.info var_ptr(%[[VAL_10]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>, volatile>, volatile>, !fir.box<!fir.ptr<!fir.array<?xi32>>, volatile>) map_clauses(always, to) capture(ByRef) members(%[[VAL_37]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("array1") -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>, volatile>, volatile>
! CHECK: %[[VAL_ATTACH_2:.*]] = omp.map.info var_ptr(%[[VAL_10]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>, volatile>, volatile>, !fir.box<!fir.ptr<!fir.array<?xi32>>, volatile>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds({{.*}}) name("array1") -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>, volatile>, volatile>
! CHECK: omp.target_enter_data map_entries(%[[VAL_38]], %[[VAL_ATTACH_2]], %[[VAL_37]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>, volatile>, volatile>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>, volatile>, volatile>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>)
diff --git a/flang/test/Transforms/omp-map-info-finalization.fir b/flang/test/Transforms/omp-map-info-finalization.fir
index e9ac6d5f3d505f..ee63c38662c7ce 100644
--- a/flang/test/Transforms/omp-map-info-finalization.fir
+++ b/flang/test/Transforms/omp-map-info-finalization.fir
@@ -31,12 +31,12 @@ func.func @test_descriptor_expansion_pass(%arg0: !fir.box<!fir.array<?xi32>>) {
// CHECK: %[[DECLARE2:.*]]:2 = hlfir.declare %[[ALLOCA2]] uniq_name("test2") fortran_attrs<allocatable> : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
// CHECK: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%{{.*}} : index) upper_bound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) start_idx(%{{.*}} : index) stride_in_bytes(true)
// CHECK: %[[BASE_ADDR_OFF:.*]] = fir.box_offset %[[DECLARE2]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
-// CHECK: %[[DESC_MEMBER_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE2]]#1 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) -> !fir.llvm_ptr<!fir.ref<i32>>
+// CHECK: %[[DESC_MEMBER_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE2]]#1 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) name("") -> !fir.llvm_ptr<!fir.ref<i32>>
// CHECK: %[[DESC_PARENT_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE2]]#1 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(always, to) capture(ByRef) members(%[[DESC_MEMBER_MAP]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.ref<!fir.box<!fir.heap<i32>>>
// CHECK: %[[ATTACH_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE2]]#1 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>, i32) -> !fir.ref<!fir.box<!fir.heap<i32>>>
// CHECK: fir.store %[[DECLARE1]]#1 to %[[ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
// CHECK: %[[BASE_ADDR_OFF_2:.*]] = fir.box_offset %[[ALLOCA]] base_addr : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-// CHECK: %[[DESC_MEMBER_MAP_2:.*]] = omp.map.info var_ptr(%[[ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(from) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR_OFF_2]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+// CHECK: %[[DESC_MEMBER_MAP_2:.*]] = omp.map.info var_ptr(%[[ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(from) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR_OFF_2]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
// CHECK: %[[DESC_PARENT_MAP_2:.*]] = omp.map.info var_ptr(%[[ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(always, to) capture(ByRef) members(%[[DESC_MEMBER_MAP_2]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.array<?xi32>>
// CHECK: %[[ATTACH_MAP_2:.*]] = omp.map.info var_ptr(%[[ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR_OFF_2]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) -> !fir.ref<!fir.array<?xi32>>
// CHECK: omp.target kernel_type(generic) map_entries(%[[DESC_PARENT_MAP]] -> %[[ARG1:.*]], %[[DESC_PARENT_MAP_2]] -> %[[ARG3:.*]], %[[ATTACH_MAP]] -> %[[ARG2:.*]], %[[ATTACH_MAP_2]] -> %[[ARG4:.*]], %[[DESC_MEMBER_MAP]] -> %[[ARG5:.*]], %[[DESC_MEMBER_MAP_2]] -> %[[ARG6:.*]] : {{.*}}) {
@@ -113,7 +113,7 @@ func.func @dtype_alloca_op_block_add(%arg0: !fir.ref<!fir.type<_QFtest_derived_t
// CHECK: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%{{.*}} : index) upper_bound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) start_idx(%{{.*}} : index) stride_in_bytes(true)
// CHECK: %[[MEMBER_COORD:.*]] = fir.coordinate_of %[[ALLOCA]]#0, array_j : (!fir.ref<[[REC_TY]]>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[MEMBER_BASE_ADDR:.*]] = fir.box_offset %[[MEMBER_COORD:.*]] base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-// CHECK: %[[MAP_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("one_l%array_j") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+// CHECK: %[[MAP_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
// CHECK: %[[MAP_MEMBER_DESCRIPTOR:.*]] = omp.map.info var_ptr(%[[MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) name("one_l%array_j") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[MAP_MEMBER_ATTACH:.*]] = omp.map.info var_ptr(%[[MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("one_l%array_j") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[MAP_MEMBER_PARENT:.*]] = omp.map.info var_ptr(%[[ALLOCA]]#0 : !fir.ref<[[REC_TY]]>>, [[REC_TY]]>) map_clauses(tofrom) capture(ByRef) members(%10, %9 : [4], [4, 0] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("one_l") partial_map(true) -> !fir.ref<[[REC_TY]]>>
@@ -154,14 +154,14 @@ func.func @alloca_dtype_map_op_block_add(%arg0 : !fir.ref<!fir.box<!fir.heap<!fi
// CHECK: %[[LOAD_ALLOCA:.*]] = fir.load %[[ALLOCA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>
// CHECK: %[[ALLOCATABLE_MEMBER_COORD:.*]] = fir.coordinate_of %[[LOAD_ALLOCA]], array_j : (!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[ALLOCATABLE_MEMBER_BASE_ADDR:.*]] = fir.box_offset %[[ALLOCATABLE_MEMBER_COORD]] base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-// CHECK: %[[MAP_ALLOCA_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[ALLOCATABLE_MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[ALLOCATABLE_MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("one_l%array_j") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+// CHECK: %[[MAP_ALLOCA_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[ALLOCATABLE_MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[ALLOCATABLE_MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
// CHECK: %[[MAP_ALLOCA_MEMBER_DESCRIPTOR:.*]] = omp.map.info var_ptr(%[[ALLOCATABLE_MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) name("one_l%array_j") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[MAP_ALLOCA_MEMBER_ATTACH:.*]] = omp.map.info var_ptr(%[[ALLOCATABLE_MEMBER_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%8) name("one_l%array_j") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[LOAD_ALLOCA2:.*]] = fir.load %[[ALLOCA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>
// CHECK: %[[REGULAR_MEMBER_COORD:.*]] = fir.coordinate_of %[[LOAD_ALLOCA2]], k : (!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) -> !fir.ref<i32>
// CHECK: %[[MAP_REGULAR_MEMBER:.*]] = omp.map.info var_ptr(%[[REGULAR_MEMBER_COORD]] : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) name("one_l%k") -> !fir.ref<i32>
// CHECK: %[[ALLOCATABLE_PARENT_BASE_ADDR:.*]] = fir.box_offset %[[ALLOCA]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>
-// CHECK: %[[MAP_ALLOCA_PARENT_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[ALLOCA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[ALLOCATABLE_PARENT_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.type<[[REC_TY]]>) name("one_l") -> !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>
+// CHECK: %[[MAP_ALLOCA_PARENT_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[ALLOCA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[ALLOCATABLE_PARENT_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.type<[[REC_TY]]>) name("") -> !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>
// CHECK: %[[MAP_PARENT_DESCRIPTOR:.*]] = omp.map.info var_ptr(%[[ALLOCA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) map_clauses(always, to) capture(ByRef) members(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : [0], [0, 4], [0, 4, 0], [0, 5] : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, !fir.ref<i32>) name("one_l") -> !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>
// CHECK: %[[ALLOCA_PARENT_ATTACH:.*]] = omp.map.info var_ptr(%[[ALLOCA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFtest_allocatable_derived_type_map_operand_and_block_additionTone_layer{i:f32,scalar:!fir.box<!fir.heap<i32>>,array_i:!fir.array<10xi32>,j:f32,array_j:!fir.box<!fir.heap<!fir.array<?xi32>>>,k:i32}>>>>, !fir.box<!fir.heap<!fir.type<_QFtest_allocatable_derived_type_map_operand_and_block_additionTone_layer{i:f32,scalar:!fir.box<!fir.heap<i32>>,array_i:!fir.array<10xi32>,j:f32,array_j:!fir.box<!fir.heap<!fir.array<?xi32>>>,k:i32}>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.type<_QFtest_allocatable_derived_type_map_operand_and_block_additionTone_layer{i:f32,scalar:!fir.box<!fir.heap<i32>>,array_i:!fir.array<10xi32>,j:f32,array_j:!fir.box<!fir.heap<!fir.array<?xi32>>>,k:i32}>>>, !fir.type<_QFtest_allocatable_derived_type_map_operand_and_block_additionTone_layer{i:f32,scalar:!fir.box<!fir.heap<i32>>,array_i:!fir.array<10xi32>,j:f32,array_j:!fir.box<!fir.heap<!fir.array<?xi32>>>,k:i32}>) name("one_l") -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QFtest_allocatable_derived_type_map_operand_and_block_additionTone_layer{i:f32,scalar:!fir.box<!fir.heap<i32>>,array_i:!fir.array<10xi32>,j:f32,array_j:!fir.box<!fir.heap<!fir.array<?xi32>>>,k:i32}>>>>
// CHECK: omp.target kernel_type(generic) map_entries(%[[MAP_PARENT_DESCRIPTOR]] -> %[[ARG1:.*]], %[[MAP_ALLOCA_MEMBER_DESCRIPTOR]] -> %[[ARG2:.*]], %[[MAP_REGULAR_MEMBER]] -> %[[ARG3:.*]], %[[MAP_ALLOCA_MEMBER_ATTACH]] -> %[[ARG4:.*]], %[[ALLOCA_PARENT_ATTACH]] -> %[[ARG5:.*]], %[[MAP_ALLOCA_PARENT_BASE_ADDR]] -> %[[ARG6:.*]], %[[MAP_ALLOCA_MEMBER_BASE_ADDR]] -> %[[ARG7:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<i32>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) {
@@ -206,7 +206,7 @@ func.func @alloca_dtype_map_op_block_add(%arg0 : !fir.ref<!fir.box<!fir.heap<!fi
// CHECK: %[[INTERMEDIATE_DTYPE_NESTED_MEMBER:.*]] = fir.coordinate_of %[[ALLOCA_LOAD]], nest : (!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) -> !fir.ref<!fir.type<[[REC_TY2:_QFtest_alloca_nested_derived_type_map_operand_and_block_additionTmiddle_layer{i:f32,array_i:!fir.array<10xi32>,array_k:!fir.box<!fir.heap<!fir.array<\?xi32>>>,k:i32}]]>>
// CHECK: %[[NESTED_ALLOCA_MEMBER:.*]] = fir.coordinate_of %[[INTERMEDIATE_DTYPE_NESTED_MEMBER]], array_k : (!fir.ref<!fir.type<[[REC_TY2]]>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[NESTED_ALLOCA_MEMBER_BASE_ADDR:.*]] = fir.box_offset %[[NESTED_ALLOCA_MEMBER]] base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-// CHECK: %[[MAP_NESTED_ALLOCA_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[NESTED_ALLOCA_MEMBER]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[NESTED_ALLOCA_MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("one_l%nest%array_k") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+// CHECK: %[[MAP_NESTED_ALLOCA_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[NESTED_ALLOCA_MEMBER]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[NESTED_ALLOCA_MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
// CHECK: %[[MAP_NESTED_ALLOCA_MEMBER:.*]] = omp.map.info var_ptr(%[[NESTED_ALLOCA_MEMBER]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) name("one_l%nest%array_k") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[MAP_NESTED_ALLOCA_MEMBER_ATTACH:.*]] = omp.map.info var_ptr(%[[NESTED_ALLOCA_MEMBER]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("one_l%nest%array_k") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[ALLOCA_LOAD2:.*]] = fir.load %[[ALLOCA]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>
@@ -214,7 +214,7 @@ func.func @alloca_dtype_map_op_block_add(%arg0 : !fir.ref<!fir.box<!fir.heap<!fi
// CHECK: %[[NESTED_REGULAR_MEMBER:.*]] = fir.coordinate_of %[[INTERMEDIATE_DTYPE_NESTED_MEMBER2]], k : (!fir.ref<!fir.type<[[REC_TY2]]>>) -> !fir.ref<i32>
// CHECK: %[[MAP_NESTED_REGULAR_MEMBER:.*]] = omp.map.info var_ptr(%[[NESTED_REGULAR_MEMBER:.*]] : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) name("one_l%nest%k") -> !fir.ref<i32>
// CHECK: %[[ALLOCATABLE_PARENT_BASE_ADDR:.*]] = fir.box_offset %[[ALLOCA]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>
-// CHECK: %[[MAP_ALLOCATABLE_PARENT_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[ALLOCA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[ALLOCATABLE_PARENT_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.type<[[REC_TY]]>) name("one_l") -> !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>
+// CHECK: %[[MAP_ALLOCATABLE_PARENT_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[ALLOCA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[ALLOCATABLE_PARENT_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.type<[[REC_TY]]>) name("") -> !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>
// CHECK: %[[MAP_ALLOCATABLE_PARENT_DESCRIPTOR:.*]] = omp.map.info var_ptr(%[[ALLOCA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) map_clauses(always, to) capture(ByRef) members(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : [0], [0, 6, 2], [0, 6, 2, 0], [0, 6, 3] : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, !fir.ref<i32>) name("one_l") -> !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>
// CHECK: %[[MAP_ALLOCATABLE_PARENT_ATTACH:.*]] = omp.map.info var_ptr(%[[ALLOCA]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.type<[[REC_TY]]>) name("one_l") -> !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>
// CHECK: omp.target kernel_type(generic) map_entries(%[[MAP_ALLOCATABLE_PARENT_DESCRIPTOR]] -> %[[ARG1:.*]], %[[MAP_NESTED_ALLOCA_MEMBER]] -> %[[ARG2:.*]], %[[MAP_NESTED_REGULAR_MEMBER]] -> %[[ARG3:.*]], %[[MAP_NESTED_ALLOCA_MEMBER_ATTACH]] -> %[[ARG4:.*]], %[[MAP_ALLOCATABLE_PARENT_ATTACH]] -> %[[ARG5:.*]], %[[MAP_ALLOCATABLE_PARENT_BASE_ADDR]] -> %[[ARG6:.*]], %[[MAP_NESTED_ALLOCA_MEMBER_BASE_ADDR]] -> %[[ARG7:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<i32>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.type<[[REC_TY]]>>>>, !fir.llvm_ptr<!fir.ref<!fir.type<[[REC_TY]]>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) {
@@ -251,7 +251,7 @@ func.func @alloca_dtype_map_op_block_add(%arg0 : !fir.ref<!fir.box<!fir.heap<!fi
// CHECK: %[[NESTED_DTYPE_COORD:.*]] = fir.coordinate_of %[[ALLOCA]]#0, nest : (!fir.ref<!fir.type<[[REC_TY]]>>) -> !fir.ref<!fir.type<[[REC_TY2:_QFtest_nested_derived_type_alloca_map_operand_and_block_additionTmiddle_layer{i:f32,array_i:!fir.array<10xi32>,array_k:!fir.box<!fir.heap<!fir.array<\?xi32>>>,k:i32}]]>>
// CHECK: %[[ALLOCATABLE_MEMBER:.*]] = fir.coordinate_of %[[NESTED_DTYPE_COORD]], array_k : (!fir.ref<!fir.type<[[REC_TY2]]>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[ALLOCATABLE_MEMBER_BASE_ADDR:.*]] = fir.box_offset %[[ALLOCATABLE_MEMBER]] base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-// CHECK: %[[MAP_ALLOCATABLE_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[ALLOCATABLE_MEMBER]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[ALLOCATABLE_MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("one_l%nest%array_k") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+// CHECK: %[[MAP_ALLOCATABLE_MEMBER_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[ALLOCATABLE_MEMBER]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[ALLOCATABLE_MEMBER_BASE_ADDR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
// CHECK: %[[MAP_ALLOCATABLE_MEMBER_DESCRIPTOR:.*]] = omp.map.info var_ptr(%[[ALLOCATABLE_MEMBER]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) name("one_l%nest%array_k") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[MAP_ALLOCATABLE_ATTACH:.*]] = omp.map.info var_ptr(%[[ALLOCATABLE_MEMBER]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%[[BOUNDS]]) name("one_l%nest%array_k") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[MAP_PARENT:.*]] = omp.map.info var_ptr(%[[ALLOCA]]#0 : !fir.ref<!fir.type<[[REC_TY]]>>, !fir.type<[[REC_TY]]>) map_clauses(tofrom) capture(ByRef) members(%{{.*}}, %{{.*}} : [6, 2], [6, 2, 0] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) name("one_l") partial_map(true) -> !fir.ref<!fir.type<[[REC_TY]]>>
@@ -285,14 +285,14 @@ func.func @alloca_dtype_map_op_block_add(%arg0 : !fir.ref<!fir.box<!fir.heap<!fi
// CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ARG0]] {{.*}} : (!fir.ref<!fir.type<[[REC_TY]]>>) -> (!fir.ref<!fir.type<[[REC_TY]]>>, !fir.ref<!fir.type<[[REC_TY]]>>)
// CHECK: %[[DESC_1:.*]] = fir.coordinate_of %[[DECLARE]]#0, vertexes : (!fir.ref<!fir.type<[[REC_TY]]>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2:_QFmaptype_nested_derived_type_member_idxTvertexes{test:i32,vertexx:!fir.box<!fir.heap<!fir.array<\?xi32>>>,vertexy:!fir.box<!fir.heap<!fir.array<\?xi32>>>}]]>>>>>
// CHECK: %[[BASE_ADDR_1:.*]] = fir.box_offset %[[DESC_1]] base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2]]>>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?x!fir.type<[[REC_TY2]]>>>>
-// CHECK: %[[BASE_ADDR_MAP_1:.*]] = omp.map.info var_ptr(%[[DESC_1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2]]>>>>>, !fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2]]>>>>) map_clauses(storage) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR_1]] : !fir.llvm_ptr<!fir.ref<!fir.array<?x!fir.type<[[REC_TY2]]>>>>, !fir.type<[[REC_TY2]]>) bounds(%{{.*}}) name("alloca_dtype%vertexes(2_8)%vertexy") -> !fir.llvm_ptr<!fir.ref<!fir.array<?x!fir.type<[[REC_TY2]]>>>>
+// CHECK: %[[BASE_ADDR_MAP_1:.*]] = omp.map.info var_ptr(%[[DESC_1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2]]>>>>>, !fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2]]>>>>) map_clauses(storage) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR_1]] : !fir.llvm_ptr<!fir.ref<!fir.array<?x!fir.type<[[REC_TY2]]>>>>, !fir.type<[[REC_TY2]]>) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?x!fir.type<[[REC_TY2]]>>>>
// CHECK: %[[DESC_MAP_1:.*]] = omp.map.info var_ptr(%[[DESC_1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2]]>>>>>, !fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2]]>>>>) map_clauses(always, to) capture(ByRef) name("alloca_dtype%vertexes(2_8)%vertexy") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2]]>>>>>
// CHECK: %[[ATTACH_MAP_1:.*]] = omp.map.info var_ptr(%[[DESC_1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2]]>>>>>, !fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2]]>>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?x!fir.type<[[REC_TY2]]>>>>, !fir.type<[[REC_TY2]]>) bounds(%{{.*}}) name("alloca_dtype%vertexes(2_8)%vertexy") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2]]>>>>>
// CHECK: %[[DESC_LD_1:.*]] = fir.load %[[DESC_1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2]]>>>>>
// CHECK: %[[MEMBER_ACCESS_1:.*]] = fir.coordinate_of %[[DESC_LD_1]], %{{.*}} : (!fir.box<!fir.heap<!fir.array<?x!fir.type<[[REC_TY2]]>>>>, index) -> !fir.ref<!fir.type<[[REC_TY2]]>>
// CHECK: %[[DESC_2:.*]] = fir.coordinate_of %[[MEMBER_ACCESS_1]], vertexy : (!fir.ref<!fir.type<[[REC_TY2]]>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[BASE_ADDR_2:.*]] = fir.box_offset %[[DESC_2]] base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-// CHECK: %[[BASE_ADDR_MAP_2:.*]] = omp.map.info var_ptr(%[[DESC_2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR_2]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("alloca_dtype%vertexes(2_8)%vertexy") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+// CHECK: %[[BASE_ADDR_MAP_2:.*]] = omp.map.info var_ptr(%[[DESC_2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR_2]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("") -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
// CHECK: %[[DESC_MAP_2:.*]] = omp.map.info var_ptr(%[[DESC_2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(always, to) capture(ByRef) name("alloca_dtype%vertexes(2_8)%vertexy") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[DESC_MAP_ATTACH:.*]] = omp.map.info var_ptr(%[[DESC_2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, i32) bounds(%{{.*}}) name("alloca_dtype%vertexes(2_8)%vertexy") -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
// CHECK: %[[TOP_PARENT_MAP:.*]] = omp.map.info var_ptr({{.*}} : {{.*}}) map_clauses(storage) capture(ByRef) members({{.*}}) name("alloca_dtype") partial_map(true) -> {{.*}}
@@ -392,7 +392,7 @@ func.func @_QPrealtest(%arg0: !fir.boxchar<1>) {
// CHECK: %[[VAL_9:.*]] = arith.subi %[[VAL_8]]#1, %[[VAL_7]] : index
// CHECK: %[[VAL_10:.*]] = omp.map.bounds lower_bound(%[[VAL_6]] : index) upper_bound(%[[VAL_9]] : index) extent(%[[VAL_8]]#1 : index) stride(%[[VAL_7]] : index) start_idx(%[[VAL_6]] : index) stride_in_bytes(true)
// CHECK: %[[VAL_12:.*]] = fir.box_offset %[[VAL_0]] base_addr : (!fir.ref<!fir.boxchar<1>>) -> !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>
-// CHECK: %[[VAL_13:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(to) capture(ByRef) var_ptr_ptr(%[[VAL_12]] : !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>, !fir.char<1,?>) bounds(%[[VAL_10]]) -> !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>
+// CHECK: %[[VAL_13:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(to) capture(ByRef) var_ptr_ptr(%[[VAL_12]] : !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>, !fir.char<1,?>) bounds(%[[VAL_10]]) name("") -> !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>
// CHECK: %[[VAL_14:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(always, to) capture(ByRef) members(%[[VAL_13]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>) -> !fir.ref<!fir.boxchar<1>>
// CHECK: %[[ATTACH_MAP:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(attach, ref_ptr, ref_ptee) capture(ByRef) var_ptr_ptr(%{{.*}} : !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>, !fir.char<1,?>) bounds(%[[VAL_10]]) -> !fir.ref<!fir.boxchar<1>>
// CHECK: omp.target kernel_type(generic) map_entries(%[[VAL_14]] -> %[[VAL_15:.*]], %[[ATTACH_MAP]] -> {{.*}}, %[[VAL_13]] -> %[[VAL_16:.*]] : !fir.ref<!fir.boxchar<1>>, !fir.ref<!fir.boxchar<1>>, !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>) private(@boxchar.privatizer %[[VAL_3]]#0 -> %[[VAL_17:.*]] [map_idx=0] : !fir.boxchar<1>) {
diff --git a/flang/test/Transforms/omp-maps-for-privatized-symbols.fir b/flang/test/Transforms/omp-maps-for-privatized-symbols.fir
index e5933a83bd6523..893265e338f7cf 100644
--- a/flang/test/Transforms/omp-maps-for-privatized-symbols.fir
+++ b/flang/test/Transforms/omp-maps-for-privatized-symbols.fir
@@ -32,8 +32,8 @@ module attributes {omp.is_target_device = false} {
}
}
// CHECK: %[[MAP0:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<i32>, i32) map_clauses(to) capture(ByRef) name("a") -> !fir.ref<i32>
-// CHECK: %[[MAP1:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(tofrom) capture(ByRef) name("simple_var") -> !fir.ref<!fir.box<!fir.heap<i32>>>
-// CHECK: %[[MAP2:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) name("fp_int") -> !fir.ref<i32>
+// CHECK: %[[MAP1:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(tofrom) capture(ByRef) -> !fir.ref<!fir.box<!fir.heap<i32>>>
+// CHECK: %[[MAP2:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<i32>, i32) map_clauses(to) capture(ByCopy) -> !fir.ref<i32>
// CHECK: omp.target kernel_type(generic) map_entries(%[[MAP0]] -> %arg0, %[[MAP1]] -> %arg1, %[[MAP2]] -> %arg2 : !fir.ref<i32>, !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<i32>)
// -----
@@ -54,7 +54,7 @@ module attributes {omp.is_target_device = false} {
}
}
// CHECK: omp.declare_mapper @[[GEN_MAPPER:_QFTgen_t_omp_default_mapper]]
-// CHECK: %[[GEN_MAP:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.type<_QFTgen_t{{.*}}>>, !fir.type<_QFTgen_t{{.*}}>) map_clauses(tofrom) capture(ByRef) mapper(@[[GEN_MAPPER]]) name("dt") -> !fir.ref<!fir.type<_QFTgen_t{{.*}}>>
+// CHECK: %[[GEN_MAP:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.type<_QFTgen_t{{.*}}>>, !fir.type<_QFTgen_t{{.*}}>) map_clauses(tofrom) capture(ByRef) mapper(@[[GEN_MAPPER]]) -> !fir.ref<!fir.type<_QFTgen_t{{.*}}>>
// CHECK: omp.target kernel_type(generic) map_entries(%[[GEN_MAP]] -> %arg0 : !fir.ref<!fir.type<_QFTgen_t{{.*}}>>)
// -----
@@ -76,5 +76,5 @@ module attributes {omp.is_target_device = false} {
}
// CHECK-NOT: omp.declare_mapper @_QFTplain_t_omp_default_mapper
-// CHECK: %[[PLAIN_MAP:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.type<_QFTplain_t{{.*}}>>, !fir.type<_QFTplain_t{{.*}}>) map_clauses(tofrom) capture(ByRef) name("dt") -> !fir.ref<!fir.type<_QFTplain_t{{.*}}>>
+// CHECK: %[[PLAIN_MAP:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.type<_QFTplain_t{{.*}}>>, !fir.type<_QFTplain_t{{.*}}>) map_clauses(tofrom) capture(ByRef) -> !fir.ref<!fir.type<_QFTplain_t{{.*}}>>
// CHECK-NOT: mapper(
diff --git a/mlir/include/mlir/Target/LLVMIR/Dialect/OpenMPCommon.h b/mlir/include/mlir/Target/LLVMIR/Dialect/OpenMPCommon.h
index d533cbe3809898..4c2da544854c61 100644
--- a/mlir/include/mlir/Target/LLVMIR/Dialect/OpenMPCommon.h
+++ b/mlir/include/mlir/Target/LLVMIR/Dialect/OpenMPCommon.h
@@ -36,8 +36,7 @@ llvm::Constant *createSourceLocStrFromLocation(Location loc,
/// Create a constant string representing the mapping information extracted from
/// the MLIR location information.
llvm::Constant *createMappingInformation(Location loc,
- llvm::OpenMPIRBuilder &builder,
- StringRef mapName = {});
+ llvm::OpenMPIRBuilder &builder);
} // namespace LLVM
} // namespace mlir
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index aa19951ab5129f..3e970ab01dd1b2 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -7413,12 +7413,6 @@ convertClauseMapFlags(omp::ClauseMapFlags mlirFlags) {
return mapType;
}
-static StringRef getMapClauseName(omp::MapInfoOp mapOp) {
- if (StringAttr name = mapOp.getNameAttr())
- return name.getValue();
- return {};
-}
-
static void collectMapDataFromMapOperands(
MapInfoData &mapData, SmallVectorImpl<Value> &mapVars,
LLVM::ModuleTranslation &moduleTranslation, DataLayout &dl,
@@ -7501,8 +7495,7 @@ static void collectMapDataFromMapOperands(
// TODO: set HasAttachPtr from Flang for pointee-storage entries.
mapData.HasAttachPtr.push_back(false);
mapData.Names.push_back(LLVM::createMappingInformation(
- mapOp.getLoc(), *moduleTranslation.getOpenMPBuilder(),
- getMapClauseName(mapOp)));
+ mapOp.getLoc(), *moduleTranslation.getOpenMPBuilder()));
mapData.DevicePointers.push_back(llvm::OpenMPIRBuilder::DeviceInfoTy::None);
if (mapOp.getMapperId())
mapData.Mappers.push_back(
@@ -7572,8 +7565,7 @@ static void collectMapDataFromMapOperands(
// TODO: set HasAttachPtr from Flang for pointee-storage entries.
mapData.HasAttachPtr.push_back(false);
mapData.Names.push_back(LLVM::createMappingInformation(
- mapOp.getLoc(), *moduleTranslation.getOpenMPBuilder(),
- getMapClauseName(mapOp)));
+ mapOp.getLoc(), *moduleTranslation.getOpenMPBuilder()));
mapData.DevicePointers.push_back(devInfoTy);
mapData.Mappers.push_back(nullptr);
mapData.IsAMapping.push_back(false);
@@ -7635,8 +7627,7 @@ static void collectMapDataFromMapOperands(
mapData.Mappers.push_back(nullptr);
}
mapData.Names.push_back(LLVM::createMappingInformation(
- mapOp.getLoc(), *moduleTranslation.getOpenMPBuilder(),
- getMapClauseName(mapOp)));
+ mapOp.getLoc(), *moduleTranslation.getOpenMPBuilder()));
mapData.DevicePointers.push_back(
isDevicePtr ? llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer
: llvm::OpenMPIRBuilder::DeviceInfoTy::Address);
@@ -8019,8 +8010,7 @@ static void mapParentWithMembers(
combinedInfo.Mappers.emplace_back(
parentMapper && !parentClause.getPartialMap() ? parentMapper : nullptr);
combinedInfo.Names.emplace_back(LLVM::createMappingInformation(
- mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder,
- getMapClauseName(cast<omp::MapInfoOp>(mapData.MapClause[mapDataIndex]))));
+ mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder));
combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIndex]);
// Calculate size of the parent object being mapped based on the
@@ -8117,9 +8107,7 @@ static void mapParentWithMembers(
combinedInfo.DevicePointers.emplace_back(
mapData.DevicePointers[mapDataIndex]);
combinedInfo.Names.emplace_back(LLVM::createMappingInformation(
- mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder,
- getMapClauseName(
- cast<omp::MapInfoOp>(mapData.MapClause[mapDataIndex]))));
+ mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder));
combinedInfo.BasePointers.emplace_back(
mapData.BasePointers[mapDataIndex]);
combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIndex]);
@@ -8161,9 +8149,7 @@ static void mapParentWithMembers(
combinedInfo.DevicePointers.emplace_back(
llvm::OpenMPIRBuilder::DeviceInfoTy::None);
combinedInfo.Names.emplace_back(LLVM::createMappingInformation(
- mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder,
- getMapClauseName(
- cast<omp::MapInfoOp>(mapData.MapClause[mapDataIndex]))));
+ mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder));
combinedInfo.BasePointers.emplace_back(
mapData.BasePointers[mapDataIndex]);
combinedInfo.Mappers.emplace_back(nullptr);
@@ -8194,9 +8180,7 @@ static void mapParentWithMembers(
combinedInfo.DevicePointers.emplace_back(
llvm::OpenMPIRBuilder::DeviceInfoTy::None);
combinedInfo.Names.emplace_back(LLVM::createMappingInformation(
- mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder,
- getMapClauseName(
- cast<omp::MapInfoOp>(mapData.MapClause[mapDataIndex]))));
+ mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder));
combinedInfo.BasePointers.emplace_back(
mapData.BasePointers[mapDataIndex]);
combinedInfo.Mappers.emplace_back(nullptr);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMPCommon.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMPCommon.cpp
index d85b2e5c0c0ea3..586b74751c3b7c 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMPCommon.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMPCommon.cpp
@@ -30,18 +30,15 @@ llvm::Constant *mlir::LLVM::createSourceLocStrFromLocation(
return builder.getOrCreateSrcLocStr(locStr, strLen);
}
-llvm::Constant *mlir::LLVM::createMappingInformation(
- Location loc, llvm::OpenMPIRBuilder &builder, StringRef mapName) {
+llvm::Constant *
+mlir::LLVM::createMappingInformation(Location loc,
+ llvm::OpenMPIRBuilder &builder) {
uint32_t strLen;
- Location childLoc = loc;
- StringRef name = mapName;
if (auto nameLoc = dyn_cast<NameLoc>(loc)) {
- childLoc = nameLoc.getChildLoc();
- if (name.empty())
- name = nameLoc.getName();
+ StringRef name = nameLoc.getName();
+ return createSourceLocStrFromLocation(nameLoc.getChildLoc(), builder, name,
+ strLen, /*forOffloadMap=*/true);
}
- if (name.empty())
- name = "unknown";
- return createSourceLocStrFromLocation(childLoc, builder, name, strLen,
+ return createSourceLocStrFromLocation(loc, builder, "unknown", strLen,
/*forOffloadMap=*/true);
}
diff --git a/mlir/test/Target/LLVMIR/omptarget-map-names.mlir b/mlir/test/Target/LLVMIR/omptarget-map-names.mlir
deleted file mode 100644
index 49fe1dfebba187..00000000000000
--- a/mlir/test/Target/LLVMIR/omptarget-map-names.mlir
+++ /dev/null
@@ -1,28 +0,0 @@
-// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
-
-// CHECK-DAG: c";x;test.f90;3;5;;\00"
-llvm.func @map_name_from_attr(%a : !llvm.ptr) {
- %m = omp.map.info var_ptr(%a : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) name("x") -> !llvm.ptr loc("test.f90":3:5)
- omp.target_data map_entries(%m : !llvm.ptr) {
- omp.terminator
- }
- llvm.return
-}
-
-// CHECK-DAG: c";y;test.f90;4;6;;\00"
-llvm.func @map_name_from_nameloc(%a : !llvm.ptr) {
- %m = omp.map.info var_ptr(%a : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) name("") -> !llvm.ptr loc("y"("test.f90":4:6))
- omp.target_data map_entries(%m : !llvm.ptr) {
- omp.terminator
- }
- llvm.return
-}
-
-// CHECK-DAG: c";unknown;test.f90;5;7;;\00"
-llvm.func @map_name_unknown(%a : !llvm.ptr) {
- %m = omp.map.info var_ptr(%a : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) name("") -> !llvm.ptr loc("test.f90":5:7)
- omp.target_data map_entries(%m : !llvm.ptr) {
- omp.terminator
- }
- llvm.return
-}
diff --git a/offload/test/offloading/fortran/target-firstprivate-info.f90 b/offload/test/offloading/fortran/target-firstprivate-info.f90
deleted file mode 100644
index c10274efcf9cc7..00000000000000
--- a/offload/test/offloading/fortran/target-firstprivate-info.f90
+++ /dev/null
@@ -1,82 +0,0 @@
-! REQUIRES: flang, amdgpu
-
-! RUN: %libomptarget-compile-fortran-generic
-! RUN: env LIBOMPTARGET_INFO=1 %libomptarget-run-generic 2>&1 | %fcheck-generic
-! RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=VALUE
-
-module fp_info_mod
- type :: point
- integer :: x
- integer :: y
- end type
-end module
-
-program target_firstprivate_info
- use fp_info_mod
- implicit none
- integer :: factor, base, res
- integer :: arr(4)
- integer, allocatable :: pa(:)
- character(len=4) :: cstr
- type(point) :: p
- factor = 7; base = 5; res = 0
- arr = (/1, 2, 3, 4/)
- p%x = 3; p%y = 4
- allocate(pa(4)); pa = 1
- cstr = "abcd"
-
- ! 'factor' is explicitly privatized, 'base' is an implicit scalar capture.
- !$omp target map(tofrom: res) firstprivate(factor)
- res = factor + base
- !$omp end target
- print *, "res1 =", res
-
- ! array firstprivate is boxed for privatization; its descriptor maps must
- ! still report the source name.
- !$omp target map(tofrom: res) firstprivate(arr)
- res = arr(1) + arr(4)
- !$omp end target
- print *, "res2 =", res
-
- ! derived-type firstprivate
- !$omp target map(tofrom: res) firstprivate(p)
- res = p%x + p%y
- !$omp end target
- print *, "res3 =", res
-
- ! a fixed-size private array is not mapped, but a private allocatable maps its
- ! descriptor (copy left unallocated/unused here) and must report its name.
- !$omp target map(tofrom: res) private(pa)
- res = 99
- !$omp end target
- print *, "res4 =", res
-
- ! character firstprivate.
- !$omp target map(tofrom: res) firstprivate(cstr)
- res = ichar(cstr(1:1))
- !$omp end target
- print *, "res5 =", res
- deallocate(pa)
-end program target_firstprivate_info
-
-! CHECK: Entering OpenMP kernel {{.*}} with 3 arguments:
-! CHECK-DAG: tofrom(res)[4]
-! CHECK-DAG: firstprivate(base)[4] (implicit)
-! CHECK-DAG: to(factor)[4]
-! Array firstprivate is boxed; the descriptor maps still name the variable.
-! CHECK: Entering OpenMP kernel {{.*}} with 5 arguments:
-! CHECK-DAG: to(arr)[48]
-! CHECK-DAG: attach(arr)[48]
-! CHECK: Entering OpenMP kernel {{.*}} with 2 arguments:
-! CHECK-DAG: tofrom(p)[8]
-! A private allocatable and a character firstprivate are named too.
-! CHECK: Entering OpenMP kernel {{.*}} with 5 arguments:
-! CHECK-DAG: to(pa)[48]
-! CHECK-DAG: attach(pa)[48]
-! CHECK: Entering OpenMP kernel {{.*}} with 2 arguments:
-! CHECK-DAG: to(cstr)[4]
-! VALUE: res1 = 12
-! VALUE: res2 = 5
-! VALUE: res3 = 7
-! VALUE: res4 = 99
-! VALUE: res5 = 97
>From 0f4847e38788c9f1d88878e1fea8c7158d7cd4a5 Mon Sep 17 00:00:00 2001
From: Andy Kaylor <akaylor at nvidia.com>
Date: Tue, 6 Oct 2026 13:21:48 -0700
Subject: [PATCH 19/46] [CIR] Add missing test dependency (#229546)
I recently made a change to introduce an explicit list of dependencies
for the check-clang-cir target. A separate change introduced a new
dependency on llvm-readobj that was not included in the list. The result
was that building the `check-clang-cir` target could leave a stale
version of llvm-readobj in place.
This change adds the missing dependency.
---
clang/test/CMakeLists.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt
index 72301580933211..d1d79c30ca9cb3 100644
--- a/clang/test/CMakeLists.txt
+++ b/clang/test/CMakeLists.txt
@@ -263,6 +263,7 @@ if(CLANG_ENABLE_CIR)
llvm-config
llvm-dis
llvm-objdump
+ llvm-readelf
not
opt
split-file
>From 35e97e88c49d4a0c7bb59d87ad60b117da2a845d Mon Sep 17 00:00:00 2001
From: "forking-google-bazel-bot[bot]"
<265904573+forking-google-bazel-bot[bot]@users.noreply.github.com>
Date: Tue, 6 Oct 2026 13:24:52 -0700
Subject: [PATCH 20/46] [Bazel] Fixes f5dcdfd (#229539)
This fixes f5dcdfdfb74f5b22c2416fd7811128dbe7c5a136 (#229463).
Buildkite error link:
https://buildkite.com/llvm-project/upstream-bazel/builds?commit=f5dcdfdfb74f5b22c2416fd7811128dbe7c5a136
Co-authored-by: Google Bazel Bot <google-bazel-bot at google.com>
---
utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 1 +
.../llvm-project-overlay/libc/test/src/__support/BUILD.bazel | 1 +
2 files changed, 2 insertions(+)
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index e68657cba201a2..89b4dde68bc3f5 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -2164,6 +2164,7 @@ libc_support_library(
":__support_cpp_type_traits",
":__support_ctype_utils",
":__support_macros_config",
+ ":__support_wctype_utils",
":hdr_stdint_proxy",
],
)
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
index 7aed992a4ad86c..a585fa1af9627f 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
@@ -70,6 +70,7 @@ libc_test(
"//libc:__support_cpp_string_view",
"//libc:__support_integer_literals",
"//libc:__support_integer_to_string",
+ "//libc:__support_macros_properties_types",
"//libc:__support_uint128",
],
)
>From 7c29e5b10efede6213a927773ff59c34fc8aedb1 Mon Sep 17 00:00:00 2001
From: Joshua Batista <jbatista at microsoft.com>
Date: Tue, 6 Oct 2026 13:40:05 -0700
Subject: [PATCH 21/46] [DirectX] Cast the pointer for atomics whose value type
differs (#226563)
`InterlockedExchange` on a `float` fails the DXIL validator with
`Invalid record`.
`DXILLegalize` rewrites a float exchange into an integer exchange, but
it does not change the pointer: Only `load`, `store` and `getelementptr`
users get their pointer elements casted.
This change adds `atomicrmw` and `cmpxchg` to PointerTypeAnalysis and
DXILPrepare. `cmpxchg` has the same defect, and float
`InterlockedCompareExchange` needs the same fix.
The output now matches DXC. If the types already agree, the atomic gets
no cast.
The test `llvm/test/tools/dxil-dis/atomicrmw.ll` covers a scalar global,
an array element and a `cmpxchg`. The `InterlockedExchange.float`
offload test now passes the validator.
Assisted by: Github Copilot
Fixes: https://github.com/llvm/llvm-project/issues/227051
---------
Copilot-Session: f3b26c5c-98e3-4091-8c86-a002c4ba46dc
---
llvm/lib/Target/DirectX/DXILPrepare.cpp | 19 ++++++++++++
.../DirectXIRPasses/PointerTypeAnalysis.cpp | 4 +++
llvm/test/tools/dxil-dis/atomicrmw.ll | 31 +++++++++++++++++--
3 files changed, 52 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/DirectX/DXILPrepare.cpp b/llvm/lib/Target/DirectX/DXILPrepare.cpp
index 23bc42e549dee0..c73c4658478ace 100644
--- a/llvm/lib/Target/DirectX/DXILPrepare.cpp
+++ b/llvm/lib/Target/DirectX/DXILPrepare.cpp
@@ -190,6 +190,25 @@ class DXILPrepareModule : public ModulePass {
GEP->setOperand(0, NoOpBitcast);
continue;
}
+ // An atomic on a float allocation exchanges the bit pattern as an
+ // integer, so the pointer element type does not match the value
+ // type. Typed pointers need a cast to keep the two in agreement.
+ if (auto *RMW = dyn_cast<AtomicRMWInst>(&I)) {
+ if (Value *NoOpBitcast = maybeGenerateBitcast(
+ Builder, PointerTypes, I, RMW->getPointerOperand(),
+ RMW->getValOperand()->getType()))
+ RMW->setOperand(AtomicRMWInst::getPointerOperandIndex(),
+ NoOpBitcast);
+ continue;
+ }
+ if (auto *CmpXchg = dyn_cast<AtomicCmpXchgInst>(&I)) {
+ if (Value *NoOpBitcast = maybeGenerateBitcast(
+ Builder, PointerTypes, I, CmpXchg->getPointerOperand(),
+ CmpXchg->getNewValOperand()->getType()))
+ CmpXchg->setOperand(AtomicCmpXchgInst::getPointerOperandIndex(),
+ NoOpBitcast);
+ continue;
+ }
}
}
}
diff --git a/llvm/lib/Target/DirectX/DirectXIRPasses/PointerTypeAnalysis.cpp b/llvm/lib/Target/DirectX/DirectXIRPasses/PointerTypeAnalysis.cpp
index c58af1560a6b78..5da22aaea3b821 100644
--- a/llvm/lib/Target/DirectX/DirectXIRPasses/PointerTypeAnalysis.cpp
+++ b/llvm/lib/Target/DirectX/DirectXIRPasses/PointerTypeAnalysis.cpp
@@ -63,6 +63,10 @@ Type *classifyPointerType(const Value *V, PointerTypeMap &Map) {
// When store value is ptr type, cannot get more type info.
if (NewPointeeTy->isPointerTy())
continue;
+ } else if (const auto *Inst = dyn_cast<AtomicRMWInst>(User)) {
+ NewPointeeTy = Inst->getValOperand()->getType();
+ } else if (const auto *Inst = dyn_cast<AtomicCmpXchgInst>(User)) {
+ NewPointeeTy = Inst->getNewValOperand()->getType();
} else if (const auto *GEP = dyn_cast<GEPOperator>(User)) {
NewPointeeTy = GEP->getSourceElementType();
}
diff --git a/llvm/test/tools/dxil-dis/atomicrmw.ll b/llvm/test/tools/dxil-dis/atomicrmw.ll
index d8edc4c68aa6ce..06b816dcb0e3a1 100644
--- a/llvm/test/tools/dxil-dis/atomicrmw.ll
+++ b/llvm/test/tools/dxil-dis/atomicrmw.ll
@@ -1,17 +1,44 @@
; RUN: llc --filetype=obj %s --stop-after=dxil-write-bitcode -o %t.bc
; RUN: dxil-dis %t.bc -o - | FileCheck %s
-; Verify that the DXIL reader supports the atomicrmw record.
+; Verify that the DXIL reader supports the atomicrmw and the cmpxchg records.
+; A float allocation needs a pointer cast, because the atomic value is an
+; integer and a DXIL pointer carries the element type.
target triple = "dxil-pc-shadermodel6.0-compute"
@gsm = internal addrspace(3) global i32 zeroinitializer, align 4
+ at gsm_arr = internal addrspace(3) global [4 x i32] zeroinitializer, align 4
+ at gsm_float = internal addrspace(3) global float zeroinitializer, align 4
+ at gsm_farr = internal addrspace(3) global [4 x float] zeroinitializer, align 4
; CHECK-LABEL: define void @main()
-; CHECK: atomicrmw add i32 addrspace(3)* @gsm, i32 1 monotonic
+; An integer allocation already agrees with the value, so it gets no cast.
+; CHECK-NEXT: atomicrmw add i32 addrspace(3)* @gsm, i32 1 monotonic
+; CHECK-NEXT: [[INDEX:%.*]] = load i32, i32 addrspace(3)* @gsm, align 4
+; CHECK-NEXT: [[INT_MUL:%.*]] = mul i32 [[INDEX]], 1
+; CHECK-NEXT: [[INT_INDEX:%.*]] = add i32 0, [[INT_MUL]]
+; CHECK-NEXT: [[INT_GEP:%.*]] = getelementptr [4 x i32], [4 x i32] addrspace(3)* @gsm_arr, i32 0, i32 [[INT_INDEX]]
+; CHECK-NEXT: atomicrmw add i32 addrspace(3)* [[INT_GEP]], i32 6 monotonic
+; CHECK-NEXT: [[P0:%.*]] = bitcast float addrspace(3)* @gsm_float to i32 addrspace(3)*
+; CHECK-NEXT: atomicrmw xchg i32 addrspace(3)* [[P0]], i32 2 monotonic
+; CHECK-NEXT: [[FLOAT_MUL:%.*]] = mul i32 [[INDEX]], 1
+; CHECK-NEXT: [[FLOAT_INDEX:%.*]] = add i32 0, [[FLOAT_MUL]]
+; CHECK-NEXT: [[FLOAT_GEP:%.*]] = getelementptr [4 x float], [4 x float] addrspace(3)* @gsm_farr, i32 0, i32 [[FLOAT_INDEX]]
+; CHECK-NEXT: [[P1:%.*]] = bitcast float addrspace(3)* [[FLOAT_GEP]] to i32 addrspace(3)*
+; CHECK-NEXT: atomicrmw xchg i32 addrspace(3)* [[P1]], i32 3 monotonic
+; CHECK-NEXT: [[P2:%.*]] = bitcast float addrspace(3)* @gsm_float to i32 addrspace(3)*
+; CHECK-NEXT: cmpxchg i32 addrspace(3)* [[P2]], i32 4, i32 5 monotonic monotonic
define void @main() #0 {
%old = atomicrmw add ptr addrspace(3) @gsm, i32 1 monotonic
+ %index = load i32, ptr addrspace(3) @gsm, align 4
+ %int_gep = getelementptr [4 x i32], ptr addrspace(3) @gsm_arr, i32 0, i32 %index
+ %arr = atomicrmw add ptr addrspace(3) %int_gep, i32 6 monotonic
+ %f = atomicrmw xchg ptr addrspace(3) @gsm_float, i32 2 monotonic
+ %gep = getelementptr [4 x float], ptr addrspace(3) @gsm_farr, i32 0, i32 %index
+ %g = atomicrmw xchg ptr addrspace(3) %gep, i32 3 monotonic
+ %c = cmpxchg ptr addrspace(3) @gsm_float, i32 4, i32 5 monotonic monotonic
ret void
}
>From 3a3b723447f94e41f9fe0da39ed015df47125469 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Tue, 6 Oct 2026 22:47:33 +0200
Subject: [PATCH 22/46] AMDGPU/GlobalISel: Split 4-byte aligned 64-bit LDS
accesses on SI (#229531)
SI cannot use ds_read2_b32/ds_write2_b32 for 4-byte aligned 64-bit
accesses due to the LDS bounds checking bug with negative base
addresses, and the selection patterns for them require
HasUsableDSOffset. The explicit legality rules still treated these as
legal, so they failed to select. Split them into 32-bit accesses as
SelectionDAG does, and enable the GFX6 run lines in the local load and
store tests.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 57 ++---
.../AMDGPU/GlobalISel/legalize-load-local.mir | 154 ++++++++-----
.../AMDGPU/GlobalISel/load-local.128.ll | 152 ++++++++++++-
.../AMDGPU/GlobalISel/load-local.96.ll | 125 ++++++++++-
.../AMDGPU/GlobalISel/store-local.128.ll | 204 +++++++++++++++++-
.../AMDGPU/GlobalISel/store-local.96.ll | 30 ++-
6 files changed, 620 insertions(+), 102 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 03987fb3f36b7a..dd2fb3a5fd7c92 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1582,6 +1582,10 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
AtomicOrdering::NotAtomic))
return true;
+ if (AS == AMDGPUAS::LOCAL_ADDRESS && MemSize == 64 &&
+ !ST.hasUsableDSOffset() && Query.MMODescrs[0].AlignInBits == 32)
+ return true;
+
// Catch weird sized loads that don't evenly divide into the access sizes
// TODO: May be able to widen depending on alignment etc.
unsigned NumRegs = (MemSize + 31) / 32;
@@ -1600,6 +1604,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
unsigned GlobalAlign32 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 32;
unsigned GlobalAlign16 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 16;
unsigned GlobalAlign8 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 8;
+ unsigned LocalAlign64 = ST.hasUsableDSOffset() ? 32 : 64;
// TODO: Refine based on subtargets which support unaligned access or 128-bit
// LDS
@@ -1611,32 +1616,32 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
auto &Actions = getActionDefinitionsBuilder(Op);
// Explicitly list some common cases.
// TODO: Does this help compile time at all?
- Actions.legalForTypesWithMemDesc({{S32, GlobalPtr, S32, GlobalAlign32},
- {V2S32, GlobalPtr, V2S32, GlobalAlign32},
- {V4S32, GlobalPtr, V4S32, GlobalAlign32},
- {S64, GlobalPtr, S64, GlobalAlign32},
- {V2S64, GlobalPtr, V2S64, GlobalAlign32},
- {V2S16, GlobalPtr, V2S16, GlobalAlign32},
- {S32, GlobalPtr, S8, GlobalAlign8},
- {S32, GlobalPtr, S16, GlobalAlign16},
-
- {S32, LocalPtr, S32, 32},
- {S64, LocalPtr, S64, 32},
- {V2S32, LocalPtr, V2S32, 32},
- {S32, LocalPtr, S8, 8},
- {S32, LocalPtr, S16, 16},
- {V2S16, LocalPtr, S32, 32},
-
- {S32, PrivatePtr, S32, 32},
- {S32, PrivatePtr, S8, 8},
- {S32, PrivatePtr, S16, 16},
- {V2S16, PrivatePtr, S32, 32},
-
- {S32, ConstantPtr, S32, GlobalAlign32},
- {V2S32, ConstantPtr, V2S32, GlobalAlign32},
- {V4S32, ConstantPtr, V4S32, GlobalAlign32},
- {S64, ConstantPtr, S64, GlobalAlign32},
- {V2S32, ConstantPtr, V2S32, GlobalAlign32}});
+ Actions.legalForTypesWithMemDesc(
+ {{S32, GlobalPtr, S32, GlobalAlign32},
+ {V2S32, GlobalPtr, V2S32, GlobalAlign32},
+ {V4S32, GlobalPtr, V4S32, GlobalAlign32},
+ {S64, GlobalPtr, S64, GlobalAlign32},
+ {V2S64, GlobalPtr, V2S64, GlobalAlign32},
+ {V2S16, GlobalPtr, V2S16, GlobalAlign32},
+ {S32, GlobalPtr, S8, GlobalAlign8},
+ {S32, GlobalPtr, S16, GlobalAlign16},
+
+ {S32, LocalPtr, S32, 32},
+ {S64, LocalPtr, S64, LocalAlign64},
+ {V2S32, LocalPtr, V2S32, LocalAlign64},
+ {S32, LocalPtr, S8, 8},
+ {S32, LocalPtr, S16, 16},
+ {V2S16, LocalPtr, S32, 32},
+
+ {S32, PrivatePtr, S32, 32},
+ {S32, PrivatePtr, S8, 8},
+ {S32, PrivatePtr, S16, 16},
+ {V2S16, PrivatePtr, S32, 32},
+
+ {S32, ConstantPtr, S32, GlobalAlign32},
+ {V2S32, ConstantPtr, V2S32, GlobalAlign32},
+ {V4S32, ConstantPtr, V4S32, GlobalAlign32},
+ {S64, ConstantPtr, S64, GlobalAlign32}});
Actions.legalForTypesWithMemDesc(ST.useRealTrue16Insts(), /* Pred */
{{S16, GlobalPtr, S8, GlobalAlign8},
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
index aaa4754a0939a2..af7ed7ad09d8e1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
@@ -2086,8 +2086,12 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i64) = G_LOAD [[COPY]](p3) :: (load (s64), align 4, addrspace 3)
- ; SI-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](i64)
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load (s32), addrspace 3)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s32) from unknown-address + 4, addrspace 3)
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; CI-LABEL: name: test_load_local_s64_align4
; CI: liveins: $vgpr0
@@ -3462,12 +3466,14 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x i32>) = G_LOAD [[COPY]](p3) :: (load (<2 x i32>), align 4, addrspace 3)
- ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p3) :: (load (i32), addrspace 3)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p3) :: (load (i32) from unknown-address + 8, addrspace 3)
- ; SI-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[LOAD]](<2 x i32>)
- ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[UV]](i32), [[UV1]](i32), [[LOAD1]](i32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p3) :: (load (i32) from unknown-address + 4, addrspace 3)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
+ ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p3) :: (load (i32) from unknown-address + 8, addrspace 3)
+ ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[LOAD]](i32), [[LOAD1]](i32), [[LOAD2]](i32)
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i96) = G_BITCAST [[BUILD_VECTOR]](<3 x i32>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](i96)
;
@@ -5129,11 +5135,18 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x i32>) = G_LOAD [[COPY]](p3) :: (load (<2 x i32>), align 4, addrspace 3)
- ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p3) :: (load (i32), addrspace 3)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(<2 x i32>) = G_LOAD [[PTR_ADD]](p3) :: (load (<2 x i32>) from unknown-address + 8, align 4, addrspace 3)
- ; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x i32>) = G_CONCAT_VECTORS [[LOAD]](<2 x i32>), [[LOAD1]](<2 x i32>)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p3) :: (load (i32) from unknown-address + 4, addrspace 3)
+ ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[LOAD]](i32), [[LOAD1]](i32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
+ ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p3) :: (load (i32) from unknown-address + 8, addrspace 3)
+ ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
+ ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p3) :: (load (i32) from unknown-address + 12, addrspace 3)
+ ; SI-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[LOAD2]](i32), [[LOAD3]](i32)
+ ; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x i32>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x i32>), [[BUILD_VECTOR1]](<2 x i32>)
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i128) = G_BITCAST [[CONCAT_VECTORS]](<4 x i32>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](i128)
;
@@ -6356,16 +6369,11 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load (s32), addrspace 3)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[LOAD]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
- ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p3) :: (load (s32) from unknown-address + 4, addrspace 3)
- ; SI-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LOAD1]](i32)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C1]](s32)
- ; SI-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL]], [[ZEXT]]
- ; SI-NEXT: [[INTTOPTR:%[0-9]+]]:_(p1) = G_INTTOPTR [[OR]](s64)
- ; SI-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p1)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s32) from unknown-address + 4, addrspace 3)
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; CI-LABEL: name: test_load_local_p1_align4
; CI: liveins: $vgpr0
@@ -12904,8 +12912,12 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), align 4, addrspace 3)
- ; SI-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>)
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load (s32), addrspace 3)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s32) from unknown-address + 4, addrspace 3)
+ ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
;
; CI-LABEL: name: test_load_local_v2s32_align4
; CI: liveins: $vgpr0
@@ -14039,12 +14051,14 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), align 4, addrspace 3)
- ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load (s32), addrspace 3)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s32) from unknown-address + 8, addrspace 3)
- ; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<2 x s32>)
- ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s32) from unknown-address + 4, addrspace 3)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
+ ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p3) :: (load (s32) from unknown-address + 8, addrspace 3)
+ ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
;
; CI-LABEL: name: test_load_local_v3s32_align4
@@ -14416,11 +14430,18 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), align 4, addrspace 3)
- ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load (s32), addrspace 3)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD]](p3) :: (load (<2 x s32>) from unknown-address + 8, align 4, addrspace 3)
- ; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[LOAD]](<2 x s32>), [[LOAD1]](<2 x s32>)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s32) from unknown-address + 4, addrspace 3)
+ ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
+ ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p3) :: (load (s32) from unknown-address + 8, addrspace 3)
+ ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
+ ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load (s32) from unknown-address + 12, addrspace 3)
+ ; SI-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD2]](s32), [[LOAD3]](s32)
+ ; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s32>), [[BUILD_VECTOR1]](<2 x s32>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x s32>)
;
; CI-LABEL: name: test_load_local_v4s32_align4
@@ -15912,11 +15933,18 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load (s64), align 4, addrspace 3)
- ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load (s32), addrspace 3)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p3) :: (load (s64) from unknown-address + 8, align 4, addrspace 3)
- ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[LOAD]](s64), [[LOAD1]](s64)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s32) from unknown-address + 4, addrspace 3)
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
+ ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p3) :: (load (s32) from unknown-address + 8, addrspace 3)
+ ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
+ ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load (s32) from unknown-address + 12, addrspace 3)
+ ; SI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; SI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
; CI-LABEL: name: test_load_local_v2s64_align4
@@ -17073,11 +17101,18 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x i32>) = G_LOAD [[COPY]](p3) :: (load (<2 x i32>), align 4, addrspace 3)
- ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p3) :: (load (i32), addrspace 3)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(<2 x i32>) = G_LOAD [[PTR_ADD]](p3) :: (load (<2 x i32>) from unknown-address + 8, align 4, addrspace 3)
- ; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x i32>) = G_CONCAT_VECTORS [[LOAD]](<2 x i32>), [[LOAD1]](<2 x i32>)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p3) :: (load (i32) from unknown-address + 4, addrspace 3)
+ ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[LOAD]](i32), [[LOAD1]](i32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
+ ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p3) :: (load (i32) from unknown-address + 8, addrspace 3)
+ ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
+ ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p3) :: (load (i32) from unknown-address + 12, addrspace 3)
+ ; SI-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[LOAD2]](i32), [[LOAD3]](i32)
+ ; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x i32>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x i32>), [[BUILD_VECTOR1]](<2 x i32>)
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p1>) = G_BITCAST [[CONCAT_VECTORS]](<4 x i32>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](<2 x p1>)
;
@@ -20168,20 +20203,23 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x i32>) = G_LOAD [[COPY]](p3) :: (load (<2 x i32>), align 4, addrspace 3)
- ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p3) :: (load (i32), addrspace 3)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p3) :: (load (i32) from unknown-address + 8, addrspace 3)
- ; SI-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[LOAD]](<2 x i32>)
- ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[UV]](i32), [[UV1]](i32), [[LOAD1]](i32)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x i32>)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 12
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p3) :: (load (i32) from unknown-address + 4, addrspace 3)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(<2 x i32>) = G_LOAD [[PTR_ADD1]](p3) :: (load (<2 x i32>) from unknown-address + 12, align 4, addrspace 3)
- ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p3) :: (load (i32) from unknown-address + 20, addrspace 3)
- ; SI-NEXT: [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[LOAD2]](<2 x i32>)
- ; SI-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[UV2]](i32), [[UV3]](i32), [[LOAD3]](i32)
+ ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p3) :: (load (i32) from unknown-address + 8, addrspace 3)
+ ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[LOAD]](i32), [[LOAD1]](i32), [[LOAD2]](i32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x i32>)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 12
+ ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
+ ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p3) :: (load (i32) from unknown-address + 12, addrspace 3)
+ ; SI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[PTR_ADD2]], [[C]](i32)
+ ; SI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p3) :: (load (i32) from unknown-address + 16, addrspace 3)
+ ; SI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[PTR_ADD2]], [[C1]](i32)
+ ; SI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p3) :: (load (i32) from unknown-address + 20, addrspace 3)
+ ; SI-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[LOAD3]](i32), [[LOAD4]](i32), [[LOAD5]](i32)
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR1]](<3 x i32>)
; SI-NEXT: [[COPY1:%[0-9]+]]:_(i96) = COPY [[BITCAST]](s96)
; SI-NEXT: [[COPY2:%[0-9]+]]:_(i96) = COPY [[BITCAST1]](s96)
@@ -20472,11 +20510,13 @@ body: |
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x i32>)
; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 12
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(<2 x i32>) = G_LOAD [[PTR_ADD1]](p3) :: (load (<2 x i32>) from unknown-address + 12, align 4, addrspace 3)
- ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p3) :: (load (i32) from unknown-address + 20, addrspace 3)
- ; SI-NEXT: [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[LOAD2]](<2 x i32>)
- ; SI-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[UV2]](i32), [[UV3]](i32), [[LOAD3]](i32)
+ ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p3) :: (load (i32) from unknown-address + 12, addrspace 3)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C2]](i32)
+ ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p3) :: (load (i32) from unknown-address + 16, addrspace 3)
+ ; SI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
+ ; SI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p3) :: (load (i32) from unknown-address + 20, addrspace 3)
+ ; SI-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[LOAD2]](i32), [[LOAD3]](i32), [[LOAD4]](i32)
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR1]](<3 x i32>)
; SI-NEXT: [[COPY1:%[0-9]+]]:_(i96) = COPY [[BITCAST]](s96)
; SI-NEXT: [[COPY2:%[0-9]+]]:_(i96) = COPY [[BITCAST1]](s96)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll
index 29f7d670cbc7b4..dfd878a4bbb7f4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll
@@ -3,9 +3,7 @@
; RUN: llc -global-isel -mtriple=amdgpu7.01-amd-amdpal < %s | FileCheck --check-prefix=GFX7 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck --check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal < %s | FileCheck --check-prefix=GFX11 %s
-
-; FIXME:
-; XUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck --check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck --check-prefix=GFX6 %s
define <4 x i32> @load_lds_v4i32(ptr addrspace(3) %ptr) {
; GFX9-LABEL: load_lds_v4i32:
@@ -36,6 +34,17 @@ define <4 x i32> @load_lds_v4i32(ptr addrspace(3) %ptr) {
; GFX11-NEXT: ds_load_b128 v[0:3], v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: load_lds_v4i32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0
+; GFX6-NEXT: ds_read_b64 v[2:3], v2
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
%load = load <4 x i32>, ptr addrspace(3) %ptr
ret <4 x i32> %load
}
@@ -240,6 +249,74 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_or3_b32 v3, v11, v12, v10
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: load_lds_v4i32_align1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 2, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 3, v0
+; GFX6-NEXT: ds_read_u8 v1, v1
+; GFX6-NEXT: ds_read_u8 v2, v2
+; GFX6-NEXT: ds_read_u8 v3, v3
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 8, v0
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 10, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 11, v0
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 12, v0
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, 13, v0
+; GFX6-NEXT: v_add_i32_e32 v9, vcc, 14, v0
+; GFX6-NEXT: v_add_i32_e32 v10, vcc, 15, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: ds_read_u8 v11, v0
+; GFX6-NEXT: ds_read_u8 v12, v4
+; GFX6-NEXT: ds_read_u8 v5, v5
+; GFX6-NEXT: ds_read_u8 v6, v6
+; GFX6-NEXT: ds_read_u8 v7, v7
+; GFX6-NEXT: ds_read_u8 v8, v8
+; GFX6-NEXT: ds_read_u8 v9, v9
+; GFX6-NEXT: ds_read_u8 v10, v10
+; GFX6-NEXT: s_waitcnt lgkmcnt(7)
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v11
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 5, v0
+; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT: ds_read_u8 v11, v4
+; GFX6-NEXT: v_or_b32_e32 v4, v2, v1
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 4, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 6, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 7, v0
+; GFX6-NEXT: ds_read_u8 v1, v1
+; GFX6-NEXT: ds_read_u8 v2, v2
+; GFX6-NEXT: ds_read_u8 v3, v3
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 9, v0
+; GFX6-NEXT: ds_read_u8 v0, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(4)
+; GFX6-NEXT: v_lshlrev_b32_e32 v11, 8, v11
+; GFX6-NEXT: s_waitcnt lgkmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v11, v1
+; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v12
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 8, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v10
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v9
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v7
+; GFX6-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX6-NEXT: v_or_b32_e32 v3, v3, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, v4
+; GFX6-NEXT: s_setpc_b64 s[30:31]
%load = load <4 x i32>, ptr addrspace(3) %ptr, align 1
ret <4 x i32> %load
}
@@ -333,6 +410,39 @@ define <4 x i32> @load_lds_v4i32_align2(ptr addrspace(3) %ptr) {
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_lshl_or_b32 v3, v8, 16, v7
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: load_lds_v4i32_align2:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 2, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_u16 v1, v1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 6, v0
+; GFX6-NEXT: ds_read_u16 v2, v2
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 4, v0
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 8, v0
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 10, v0
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 12, v0
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 14, v0
+; GFX6-NEXT: ds_read_u16 v3, v3
+; GFX6-NEXT: ds_read_u16 v0, v0
+; GFX6-NEXT: ds_read_u16 v4, v4
+; GFX6-NEXT: ds_read_u16 v5, v5
+; GFX6-NEXT: ds_read_u16 v6, v6
+; GFX6-NEXT: ds_read_u16 v7, v7
+; GFX6-NEXT: s_waitcnt lgkmcnt(7)
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: s_waitcnt lgkmcnt(4)
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX6-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v7
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT: v_or_b32_e32 v3, v3, v6
+; GFX6-NEXT: s_setpc_b64 s[30:31]
%load = load <4 x i32>, ptr addrspace(3) %ptr, align 2
ret <4 x i32> %load
}
@@ -374,6 +484,20 @@ define <4 x i32> @load_lds_v4i32_align4(ptr addrspace(3) %ptr) {
; GFX11-NEXT: ds_load_2addr_b32 v[2:3], v2 offset0:2 offset1:3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: load_lds_v4i32_align4:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 4, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 12, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b32 v0, v0
+; GFX6-NEXT: ds_read_b32 v1, v1
+; GFX6-NEXT: ds_read_b32 v2, v2
+; GFX6-NEXT: ds_read_b32 v3, v3
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
%load = load <4 x i32>, ptr addrspace(3) %ptr, align 4
ret <4 x i32> %load
}
@@ -409,6 +533,17 @@ define <4 x i32> @load_lds_v4i32_align8(ptr addrspace(3) %ptr) {
; GFX11-NEXT: ds_load_2addr_b64 v[0:3], v0 offset1:1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: load_lds_v4i32_align8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0
+; GFX6-NEXT: ds_read_b64 v[2:3], v2
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
%load = load <4 x i32>, ptr addrspace(3) %ptr, align 8
ret <4 x i32> %load
}
@@ -442,6 +577,17 @@ define <4 x i32> @load_lds_v4i32_align16(ptr addrspace(3) %ptr) {
; GFX11-NEXT: ds_load_b128 v[0:3], v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: load_lds_v4i32_align16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0
+; GFX6-NEXT: ds_read_b64 v[2:3], v2
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
%load = load <4 x i32>, ptr addrspace(3) %ptr, align 16
ret <4 x i32> %load
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll
index e644381b64e03f..38d9dc11fe38d9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll
@@ -3,9 +3,7 @@
; RUN: llc -global-isel -mtriple=amdgpu7.01-amd-amdpal < %s | FileCheck --check-prefix=GFX7 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck --check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal < %s | FileCheck --check-prefix=GFX11 %s
-
-; FIXME:
-; XUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck --check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck --check-prefix=GFX6 %s
define <3 x i32> @load_lds_v3i32(ptr addrspace(3) %ptr) {
; GFX9-LABEL: load_lds_v3i32:
@@ -36,6 +34,17 @@ define <3 x i32> @load_lds_v3i32(ptr addrspace(3) %ptr) {
; GFX11-NEXT: ds_load_b96 v[0:2], v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: load_lds_v3i32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0
+; GFX6-NEXT: ds_read_b32 v2, v2
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
%load = load <3 x i32>, ptr addrspace(3) %ptr
ret <3 x i32> %load
}
@@ -198,6 +207,56 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_or3_b32 v2, v8, v9, v7
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: load_lds_v3i32_align1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_u8 v1, v1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 2, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 3, v0
+; GFX6-NEXT: ds_read_u8 v2, v2
+; GFX6-NEXT: ds_read_u8 v3, v3
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 4, v0
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 6, v0
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 7, v0
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 8, v0
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, 9, v0
+; GFX6-NEXT: v_add_i32_e32 v9, vcc, 10, v0
+; GFX6-NEXT: v_add_i32_e32 v10, vcc, 11, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT: ds_read_u8 v11, v0
+; GFX6-NEXT: ds_read_u8 v4, v4
+; GFX6-NEXT: ds_read_u8 v5, v5
+; GFX6-NEXT: ds_read_u8 v6, v6
+; GFX6-NEXT: ds_read_u8 v7, v7
+; GFX6-NEXT: ds_read_u8 v8, v8
+; GFX6-NEXT: ds_read_u8 v9, v9
+; GFX6-NEXT: ds_read_u8 v10, v10
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 5, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(7)
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v11
+; GFX6-NEXT: ds_read_u8 v11, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 24, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 8, v11
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v10
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v9
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v7
+; GFX6-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT: s_setpc_b64 s[30:31]
%load = load <3 x i32>, ptr addrspace(3) %ptr, align 1
ret <3 x i32> %load
}
@@ -274,6 +333,32 @@ define <3 x i32> @load_lds_v3i32_align2(ptr addrspace(3) %ptr) {
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_lshl_or_b32 v2, v6, 16, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: load_lds_v3i32_align2:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 2, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 4, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 6, v0
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 8, v0
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 10, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_u16 v0, v0
+; GFX6-NEXT: ds_read_u16 v1, v1
+; GFX6-NEXT: ds_read_u16 v2, v2
+; GFX6-NEXT: ds_read_u16 v3, v3
+; GFX6-NEXT: ds_read_u16 v4, v4
+; GFX6-NEXT: ds_read_u16 v5, v5
+; GFX6-NEXT: s_waitcnt lgkmcnt(4)
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT: s_setpc_b64 s[30:31]
%load = load <3 x i32>, ptr addrspace(3) %ptr, align 2
ret <3 x i32> %load
}
@@ -315,6 +400,18 @@ define <3 x i32> @load_lds_v3i32_align4(ptr addrspace(3) %ptr) {
; GFX11-NEXT: ds_load_b32 v2, v2 offset:8
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: load_lds_v3i32_align4:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 4, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_b32 v0, v0
+; GFX6-NEXT: ds_read_b32 v1, v1
+; GFX6-NEXT: ds_read_b32 v2, v2
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
%load = load <3 x i32>, ptr addrspace(3) %ptr, align 4
ret <3 x i32> %load
}
@@ -356,6 +453,17 @@ define <3 x i32> @load_lds_v3i32_align8(ptr addrspace(3) %ptr) {
; GFX11-NEXT: ds_load_b32 v2, v2 offset:8
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: load_lds_v3i32_align8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0
+; GFX6-NEXT: ds_read_b32 v2, v2
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
%load = load <3 x i32>, ptr addrspace(3) %ptr, align 8
ret <3 x i32> %load
}
@@ -389,6 +497,17 @@ define <3 x i32> @load_lds_v3i32_align16(ptr addrspace(3) %ptr) {
; GFX11-NEXT: ds_load_b96 v[0:2], v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-LABEL: load_lds_v3i32_align16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0
+; GFX6-NEXT: ds_read_b32 v2, v2
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
%load = load <3 x i32>, ptr addrspace(3) %ptr, align 16
ret <3 x i32> %load
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
index f6517777ee2459..0892ccea7d226b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
@@ -3,9 +3,7 @@
; RUN: llc -global-isel -mtriple=amdgpu7.01-amd-amdpal < %s | FileCheck --check-prefix=GFX7 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck --check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal < %s | FileCheck --check-prefix=GFX11 %s
-
-; FIXME:
-; XUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck --check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck --check-prefix=GFX6 %s
define amdgpu_kernel void @store_lds_v4i32(ptr addrspace(3) %out, <4 x i32> %x) {
; GFX9-LABEL: store_lds_v4i32:
@@ -60,6 +58,23 @@ define amdgpu_kernel void @store_lds_v4i32(ptr addrspace(3) %out, <4 x i32> %x)
; GFX11-NEXT: v_mov_b32_e32 v4, s4
; GFX11-NEXT: ds_store_b128 v4, v[0:3]
; GFX11-NEXT: s_endpgm
+;
+; GFX6-LABEL: store_lds_v4i32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
+; GFX6-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GFX6-NEXT: s_add_u32 s0, s4, 8
+; GFX6-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
+; GFX6-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-NEXT: s_endpgm
store <4 x i32> %x, ptr addrspace(3) %out
ret void
}
@@ -287,6 +302,89 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: ds_store_b8 v5, v3 offset:14
; GFX11-NEXT: ds_store_b8_d16_hi v5, v3 offset:15
; GFX11-NEXT: s_endpgm
+;
+; GFX6-LABEL: store_lds_v4i32_align1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
+; GFX6-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_bfe_u32 s7, s0, 0x80008
+; GFX6-NEXT: s_add_u32 s8, s4, 1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s4
+; GFX6-NEXT: s_lshr_b32 s5, s0, 16
+; GFX6-NEXT: s_add_u32 s6, s4, 2
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s7
+; GFX6-NEXT: v_mov_b32_e32 v1, s8
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: s_lshr_b32 s0, s0, 24
+; GFX6-NEXT: s_add_u32 s7, s4, 3
+; GFX6-NEXT: v_mov_b32_e32 v0, s5
+; GFX6-NEXT: v_mov_b32_e32 v1, s6
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s7
+; GFX6-NEXT: s_add_u32 s0, s4, 4
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: s_bfe_u32 s7, s1, 0x80008
+; GFX6-NEXT: s_add_u32 s8, s4, 5
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: s_lshr_b32 s5, s1, 16
+; GFX6-NEXT: s_add_u32 s6, s4, 6
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s7
+; GFX6-NEXT: v_mov_b32_e32 v1, s8
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: s_lshr_b32 s0, s1, 24
+; GFX6-NEXT: s_add_u32 s1, s4, 7
+; GFX6-NEXT: v_mov_b32_e32 v0, s5
+; GFX6-NEXT: v_mov_b32_e32 v1, s6
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: s_add_u32 s0, s4, 8
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: s_bfe_u32 s6, s2, 0x80008
+; GFX6-NEXT: s_add_u32 s7, s4, 9
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: s_lshr_b32 s1, s2, 16
+; GFX6-NEXT: s_add_u32 s5, s4, 10
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s6
+; GFX6-NEXT: v_mov_b32_e32 v1, s7
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: s_lshr_b32 s0, s2, 24
+; GFX6-NEXT: s_add_u32 s2, s4, 11
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_mov_b32_e32 v1, s5
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: s_add_u32 s0, s4, 12
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: s_bfe_u32 s5, s3, 0x80008
+; GFX6-NEXT: s_add_u32 s6, s4, 13
+; GFX6-NEXT: v_mov_b32_e32 v0, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: s_lshr_b32 s1, s3, 16
+; GFX6-NEXT: s_add_u32 s2, s4, 14
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s5
+; GFX6-NEXT: v_mov_b32_e32 v1, s6
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: s_lshr_b32 s0, s3, 24
+; GFX6-NEXT: s_add_u32 s3, s4, 15
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
+; GFX6-NEXT: ds_write_b8 v1, v0
+; GFX6-NEXT: s_endpgm
store <4 x i32> %x, ptr addrspace(3) %out, align 1
ret void
}
@@ -402,6 +500,49 @@ define amdgpu_kernel void @store_lds_v4i32_align2(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: ds_store_b16 v1, v4 offset:12
; GFX11-NEXT: ds_store_b16 v1, v8 offset:14
; GFX11-NEXT: s_endpgm
+;
+; GFX6-LABEL: store_lds_v4i32_align2:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
+; GFX6-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_lshr_b32 s5, s0, 16
+; GFX6-NEXT: s_add_u32 s6, s4, 2
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s4
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s5
+; GFX6-NEXT: v_mov_b32_e32 v1, s6
+; GFX6-NEXT: s_add_u32 s0, s4, 4
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_lshr_b32 s5, s1, 16
+; GFX6-NEXT: s_add_u32 s6, s4, 6
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s5
+; GFX6-NEXT: v_mov_b32_e32 v1, s6
+; GFX6-NEXT: s_add_u32 s0, s4, 8
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_lshr_b32 s1, s2, 16
+; GFX6-NEXT: s_add_u32 s5, s4, 10
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_mov_b32_e32 v1, s5
+; GFX6-NEXT: s_add_u32 s0, s4, 12
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_lshr_b32 s1, s3, 16
+; GFX6-NEXT: s_add_u32 s2, s4, 14
+; GFX6-NEXT: v_mov_b32_e32 v0, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: ds_write_b16 v1, v0
+; GFX6-NEXT: s_endpgm
store <4 x i32> %x, ptr addrspace(3) %out, align 2
ret void
}
@@ -463,6 +604,29 @@ define amdgpu_kernel void @store_lds_v4i32_align4(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: ds_store_2addr_b32 v1, v0, v2 offset1:1
; GFX11-NEXT: ds_store_2addr_b32 v1, v3, v4 offset0:2 offset1:3
; GFX11-NEXT: s_endpgm
+;
+; GFX6-LABEL: store_lds_v4i32_align4:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
+; GFX6-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s4
+; GFX6-NEXT: s_add_u32 s0, s4, 4
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: s_add_u32 s0, s4, 8
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: s_add_u32 s0, s4, 12
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: s_endpgm
store <4 x i32> %x, ptr addrspace(3) %out, align 4
ret void
}
@@ -521,6 +685,23 @@ define amdgpu_kernel void @store_lds_v4i32_align8(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: v_mov_b32_e32 v4, s4
; GFX11-NEXT: ds_store_2addr_b64 v4, v[0:1], v[2:3] offset1:1
; GFX11-NEXT: s_endpgm
+;
+; GFX6-LABEL: store_lds_v4i32_align8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
+; GFX6-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GFX6-NEXT: s_add_u32 s0, s4, 8
+; GFX6-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
+; GFX6-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-NEXT: s_endpgm
store <4 x i32> %x, ptr addrspace(3) %out, align 8
ret void
}
@@ -578,6 +759,23 @@ define amdgpu_kernel void @store_lds_v4i32_align16(ptr addrspace(3) %out, <4 x i
; GFX11-NEXT: v_mov_b32_e32 v4, s4
; GFX11-NEXT: ds_store_b128 v4, v[0:3]
; GFX11-NEXT: s_endpgm
+;
+; GFX6-LABEL: store_lds_v4i32_align16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
+; GFX6-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GFX6-NEXT: s_add_u32 s0, s4, 8
+; GFX6-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
+; GFX6-NEXT: ds_write_b64 v2, v[0:1]
+; GFX6-NEXT: s_endpgm
store <4 x i32> %x, ptr addrspace(3) %out, align 16
ret void
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
index 6d0a59de64f3b7..3de59c7c3b526d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
@@ -3,9 +3,7 @@
; RUN: llc -global-isel -mtriple=amdgpu7.01-amd-amdpal < %s | FileCheck --check-prefix=GFX7 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck --check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal < %s | FileCheck --check-prefix=GFX11 %s
-
-; FIXME:
-; XUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck --check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck --check-prefix=GFX6 %s
define amdgpu_kernel void @store_lds_v3i32(ptr addrspace(3) %out, <3 x i32> %x) {
; GFX9-LABEL: store_lds_v3i32:
@@ -60,6 +58,7 @@ define amdgpu_kernel void @store_lds_v3i32(ptr addrspace(3) %out, <3 x i32> %x)
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: ds_store_b96 v3, v[0:2]
; GFX11-NEXT: s_endpgm
+;
; GFX6-LABEL: store_lds_v3i32:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
@@ -260,6 +259,7 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: ds_store_b8_d16_hi v6, v2 offset:10
; GFX11-NEXT: ds_store_b8_d16_hi v6, v5 offset:11
; GFX11-NEXT: s_endpgm
+;
; GFX6-LABEL: store_lds_v3i32_align1:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
@@ -424,6 +424,7 @@ define amdgpu_kernel void @store_lds_v3i32_align2(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: ds_store_b16 v1, v4 offset:8
; GFX11-NEXT: ds_store_b16 v1, v6 offset:10
; GFX11-NEXT: s_endpgm
+;
; GFX6-LABEL: store_lds_v3i32_align2:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
@@ -519,18 +520,25 @@ define amdgpu_kernel void @store_lds_v3i32_align4(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: ds_store_2addr_b32 v1, v0, v2 offset1:1
; GFX11-NEXT: ds_store_b32 v1, v3 offset:8
; GFX11-NEXT: s_endpgm
+;
; GFX6-LABEL: store_lds_v3i32_align4:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_load_dword s3, s[4:5], 0x0
; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s6
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
-; GFX6-NEXT: v_mov_b32_e32 v2, s0
-; GFX6-NEXT: ds_write_b32 v0, v1 offset:8
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: ds_write2_b32 v0, v2, v1 offset1:1
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
+; GFX6-NEXT: s_add_u32 s0, s3, 4
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: s_add_u32 s0, s3, 8
+; GFX6-NEXT: ds_write_b32 v1, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: ds_write_b32 v1, v0
; GFX6-NEXT: s_endpgm
store <3 x i32> %x, ptr addrspace(3) %out, align 4
ret void
@@ -593,6 +601,7 @@ define amdgpu_kernel void @store_lds_v3i32_align8(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: ds_store_2addr_b32 v1, v0, v2 offset1:1
; GFX11-NEXT: ds_store_b32 v1, v3 offset:8
; GFX11-NEXT: s_endpgm
+;
; GFX6-LABEL: store_lds_v3i32_align8:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
@@ -666,6 +675,7 @@ define amdgpu_kernel void @store_lds_v3i32_align16(ptr addrspace(3) %out, <3 x i
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: ds_store_b96 v3, v[0:2]
; GFX11-NEXT: s_endpgm
+;
; GFX6-LABEL: store_lds_v3i32_align16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
>From eef2584fb4613d4f77c5cd4911facf32ba496f9a Mon Sep 17 00:00:00 2001
From: Ambrose Leeb <aleeb at nvidia.com>
Date: Tue, 6 Oct 2026 22:53:38 +0200
Subject: [PATCH 23/46] [Clang] [NFC] Introduce factory functions for `Token`
(#229254)
There are quite a lot of places in Clang where we create a new `Token`
variable and then call `startToken()`, `setKind()` etc.
This patch introduces `Token::create()` to simplify creating tokens
manually, as well as `createAnnotation()` and `createEof()` because
creating annotation tokens and EOF tokens is a fairly common operation.
I made these factory functions rather than constructors because there
are a few places where we allocate uninitialised tokens. A refactor that
moves `startToken()` into a default constructor of `Token` would be
possible I think, but that would require some more changes, and I think
this is already enough of an improvement.
Assisted-by: Codex
---
clang/include/clang/Lex/Token.h | 28 +++
.../include/clang/Lex/VariadicMacroSupport.h | 3 +-
clang/lib/Format/FormatTokenSource.h | 3 +-
clang/lib/Lex/Lexer.cpp | 5 +-
clang/lib/Lex/PPDirectives.cpp | 19 +-
clang/lib/Lex/PPMacroExpansion.cpp | 28 +--
clang/lib/Lex/Pragma.cpp | 16 +-
clang/lib/Lex/TokenLexer.cpp | 6 +-
clang/lib/Parse/ParseCXXInlineMethods.cpp | 26 +--
clang/lib/Parse/ParseDecl.cpp | 11 +-
clang/lib/Parse/ParseDeclCXX.cpp | 21 +-
clang/lib/Parse/ParseExpr.cpp | 18 +-
clang/lib/Parse/ParseExprCXX.cpp | 6 +-
clang/lib/Parse/ParseObjc.cpp | 6 +-
clang/lib/Parse/ParsePragma.cpp | 220 ++++++------------
clang/lib/Parse/Parser.cpp | 3 +-
clang/lib/Rewrite/TokenRewriter.cpp | 8 +-
17 files changed, 145 insertions(+), 282 deletions(-)
diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h
index 9904b271c200ec..0e239a3571e6b2 100644
--- a/clang/include/clang/Lex/Token.h
+++ b/clang/include/clang/Lex/Token.h
@@ -192,6 +192,34 @@ class Token {
Loc = SourceLocation().getRawEncoding();
}
+ static Token create(tok::TokenKind Kind, SourceLocation Loc,
+ unsigned Length = 0) {
+ Token Tok;
+ Tok.startToken();
+ Tok.setKind(Kind);
+ Tok.setLocation(Loc);
+ Tok.setLength(Length);
+ return Tok;
+ }
+
+ static Token createAnnotation(tok::TokenKind Kind, SourceRange Range,
+ void *Value = nullptr) {
+ assert(tok::isAnnotation(Kind) && "Expected an annotation token kind");
+ Token Tok;
+ Tok.startToken();
+ Tok.setKind(Kind);
+ Tok.setAnnotationRange(Range);
+ Tok.setAnnotationValue(Value);
+ return Tok;
+ }
+
+ static Token createEof(SourceLocation Loc = SourceLocation(),
+ const void *Data = nullptr) {
+ Token Tok = create(tok::eof, Loc);
+ Tok.setEofData(Data);
+ return Tok;
+ }
+
bool hasPtrData() const { return PtrData != nullptr; }
IdentifierInfo *getIdentifierInfo() const {
diff --git a/clang/include/clang/Lex/VariadicMacroSupport.h b/clang/include/clang/Lex/VariadicMacroSupport.h
index cf86a00c6d6637..a92c1275abb403 100644
--- a/clang/include/clang/Lex/VariadicMacroSupport.h
+++ b/clang/include/clang/Lex/VariadicMacroSupport.h
@@ -157,8 +157,7 @@ namespace clang {
: VAOptDefinitionContext(PP), LeadingSpaceForStringifiedToken(false),
StringifyBefore(false), CharifyBefore(false),
BeginsWithPlaceholder(false), EndsWithPlaceholder(false) {
- SyntheticEOFToken.startToken();
- SyntheticEOFToken.setKind(tok::eof);
+ SyntheticEOFToken = Token::createEof();
}
void reset() {
diff --git a/clang/lib/Format/FormatTokenSource.h b/clang/lib/Format/FormatTokenSource.h
index 8f00e5f4582c68..d7cda33f18cbea 100644
--- a/clang/lib/Format/FormatTokenSource.h
+++ b/clang/lib/Format/FormatTokenSource.h
@@ -195,8 +195,7 @@ class ScopedMacroState : public FormatTokenSource {
: Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
Token(nullptr), PreviousToken(nullptr) {
- FakeEOF.Tok.startToken();
- FakeEOF.Tok.setKind(tok::eof);
+ FakeEOF.Tok = Token::createEof();
TokenSource = this;
Line.Level = 0;
Line.InPPDirective = true;
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 4762b38e67c00a..0abc6a56c44383 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -4663,11 +4663,8 @@ bool Lexer::LexTokenInternal(Token &Result) {
const char *Lexer::convertDependencyDirectiveToken(
const dependency_directives_scan::Token &DDTok, Token &Result) {
const char *TokPtr = BufferStart + DDTok.Offset;
- Result.startToken();
- Result.setLocation(getSourceLocation(TokPtr));
- Result.setKind(DDTok.Kind);
+ Result = Token::create(DDTok.Kind, getSourceLocation(TokPtr), DDTok.Length);
Result.setFlag((Token::TokenFlags)DDTok.Flags);
- Result.setLength(DDTok.Length);
if (Result.is(tok::raw_identifier))
Result.setRawIdentifierData(TokPtr);
else if (Result.isLiteral())
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index f1e9aaa72ff048..0797b1caf59547 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -1997,11 +1997,7 @@ void Preprocessor::EnterAnnotationToken(SourceRange Range,
// FIXME: Produce this as the current token directly, rather than
// allocating a new token for it.
auto Tok = std::make_unique<Token[]>(1);
- Tok[0].startToken();
- Tok[0].setKind(Kind);
- Tok[0].setLocation(Range.getBegin());
- Tok[0].setAnnotationEndLoc(Range.getEnd());
- Tok[0].setAnnotationValue(AnnotationVal);
+ Tok[0] = Token::createAnnotation(Kind, Range, AnnotationVal);
EnterTokenStream(std::move(Tok), 1, true, /*IsReinject*/ false);
}
@@ -4063,10 +4059,7 @@ void Preprocessor::HandleEmbedDirectiveImpl(
Data->BinaryData = BinaryContents;
Data->FileName = FileName;
- Toks[CurIdx].startToken();
- Toks[CurIdx].setKind(tok::annot_embed);
- Toks[CurIdx].setAnnotationRange(HashLoc);
- Toks[CurIdx++].setAnnotationValue(Data);
+ Toks[CurIdx++] = Token::createAnnotation(tok::annot_embed, HashLoc, Data);
// Now add the suffix tokens, if any.
if (Params.MaybeSuffixParam) {
@@ -4292,12 +4285,8 @@ void Preprocessor::HandleCXXImportDirective(Token ImportTok) {
case ImportAction::ModuleBegin:
// Let the parser know we're textually entering the module.
- DirToks.emplace_back();
- DirToks.back().startToken();
- DirToks.back().setKind(tok::annot_module_begin);
- DirToks.back().setLocation(SemiLoc);
- DirToks.back().setAnnotationEndLoc(SemiLoc);
- DirToks.back().setAnnotationValue(Action.ModuleForHeader);
+ DirToks.push_back(Token::createAnnotation(
+ tok::annot_module_begin, SemiLoc, Action.ModuleForHeader));
[[fallthrough]];
case ImportAction::ModuleImport:
diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp
index f31ee308b4cda8..5b6efa37d6fc1e 100644
--- a/clang/lib/Lex/PPMacroExpansion.cpp
+++ b/clang/lib/Lex/PPMacroExpansion.cpp
@@ -676,10 +676,8 @@ static bool GenerateNewArgTokens(Preprocessor &PP,
// Add left paren
if (FoundSeparatorToken) {
- TempToken.startToken();
- TempToken.setKind(tok::l_paren);
- TempToken.setLocation(ArgStartIterator->getLocation());
- TempToken.setLength(0);
+ TempToken =
+ Token::create(tok::l_paren, ArgStartIterator->getLocation());
NewTokens.push_back(TempToken);
}
@@ -689,10 +687,7 @@ static bool GenerateNewArgTokens(Preprocessor &PP,
// Add right paren and store the paren locations in ParenHints
if (FoundSeparatorToken) {
SourceLocation Loc = PP.getLocForEndOfToken((I - 1)->getLocation());
- TempToken.startToken();
- TempToken.setKind(tok::r_paren);
- TempToken.setLocation(Loc);
- TempToken.setLength(0);
+ TempToken = Token::create(tok::r_paren, Loc);
NewTokens.push_back(TempToken);
ParenHints.push_back(SourceRange(ArgStartIterator->getLocation(),
Loc));
@@ -856,11 +851,7 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName,
}
// Add a marker EOF token to the end of the token list for this argument.
- Token EOFTok;
- EOFTok.startToken();
- EOFTok.setKind(tok::eof);
- EOFTok.setLocation(Tok.getLocation());
- EOFTok.setLength(0);
+ Token EOFTok = Token::createEof(Tok.getLocation());
ArgTokens.push_back(EOFTok);
++NumActuals;
if (!ContainsCodeCompletionTok && NumFixedArgsLeft != 0)
@@ -917,11 +908,7 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName,
if (ContainsCodeCompletionTok) {
// Recover from not-fully-formed macro invocation during code-completion.
- Token EOFTok;
- EOFTok.startToken();
- EOFTok.setKind(tok::eof);
- EOFTok.setLocation(Tok.getLocation());
- EOFTok.setLength(0);
+ Token EOFTok = Token::createEof(Tok.getLocation());
for (; NumActuals < MinArgsExpected; ++NumActuals)
ArgTokens.push_back(EOFTok);
}
@@ -980,10 +967,7 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName,
// Add a marker EOF token to the end of the token list for this argument.
SourceLocation EndLoc = Tok.getLocation();
- Tok.startToken();
- Tok.setKind(tok::eof);
- Tok.setLocation(EndLoc);
- Tok.setLength(0);
+ Tok = Token::createEof(EndLoc);
ArgTokens.push_back(Tok);
// If we expect two arguments, add both as empty.
diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp
index 9e9f1d21980e37..078842ab08c5ef 100644
--- a/clang/lib/Lex/Pragma.cpp
+++ b/clang/lib/Lex/Pragma.cpp
@@ -1115,19 +1115,15 @@ struct PragmaDebugHandler : public PragmaHandler {
LLVM_BUILTIN_TRAP;
} else if (II->isStr("parser_crash")) {
if (!PP.getPreprocessorOpts().DisablePragmaDebugCrash) {
- Token Crasher;
- Crasher.startToken();
- Crasher.setKind(tok::annot_pragma_parser_crash);
- Crasher.setAnnotationRange(SourceRange(Tok.getLocation()));
+ Token Crasher = Token::createAnnotation(tok::annot_pragma_parser_crash,
+ Tok.getLocation());
PP.EnterToken(Crasher, /*IsReinject*/ false);
}
} else if (II->isStr("sleep")) {
std::this_thread::sleep_for(std::chrono::milliseconds(100));
} else if (II->isStr("dump")) {
- Token DumpAnnot;
- DumpAnnot.startToken();
- DumpAnnot.setKind(tok::annot_pragma_dump);
- DumpAnnot.setAnnotationRange(SourceRange(Tok.getLocation()));
+ Token DumpAnnot =
+ Token::createAnnotation(tok::annot_pragma_dump, Tok.getLocation());
PP.EnterToken(DumpAnnot, /*IsReinject*/false);
} else if (II->isStr("diag_mapping")) {
Token DiagName;
@@ -1287,9 +1283,7 @@ struct PragmaDebugHandler : public PragmaHandler {
SourceLocation NameLoc = Tok.getLocation();
MutableArrayRef<Token> Toks(
PP.getPreprocessorAllocator().Allocate<Token>(1), 1);
- Toks[0].startToken();
- Toks[0].setKind(tok::annot_pragma_captured);
- Toks[0].setLocation(NameLoc);
+ Toks[0] = Token::createAnnotation(tok::annot_pragma_captured, NameLoc);
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
/*IsReinject=*/false);
diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp
index 9b18c31e19eb17..1e6132887e319c 100644
--- a/clang/lib/Lex/TokenLexer.cpp
+++ b/clang/lib/Lex/TokenLexer.cpp
@@ -838,11 +838,9 @@ bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
// Common paste case: identifier+identifier = identifier. Avoid creating
// a lexer and other overhead.
PP.IncrementPasteCounter(true);
- Result.startToken();
- Result.setKind(tok::raw_identifier);
+ Result =
+ Token::create(tok::raw_identifier, ResultTokLoc, LHSLen + RHSLen);
Result.setRawIdentifierData(ResultTokStrPtr);
- Result.setLocation(ResultTokLoc);
- Result.setLength(LHSLen+RHSLen);
} else {
PP.IncrementPasteCounter(false);
diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp
index 35dbec8dfb2f7c..fb28b6aeb06af3 100644
--- a/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -254,11 +254,7 @@ void Parser::ParseCXXNonStaticMemberInitializer(Decl *VarD) {
// Store an artificial EOF token to ensure that we don't run off the end of
// the initializer when we come to parse it.
- Token Eof;
- Eof.startToken();
- Eof.setKind(tok::eof);
- Eof.setLocation(Tok.getLocation());
- Eof.setEofData(VarD);
+ Token Eof = Token::createEof(Tok.getLocation(), VarD);
Toks.push_back(Eof);
}
@@ -402,11 +398,8 @@ void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) {
// Mark the end of the default argument so that we know when to stop when
// we parse it later on.
Token LastDefaultArgToken = Toks->back();
- Token DefArgEnd;
- DefArgEnd.startToken();
- DefArgEnd.setKind(tok::eof);
- DefArgEnd.setLocation(LastDefaultArgToken.getEndLoc());
- DefArgEnd.setEofData(Param);
+ Token DefArgEnd =
+ Token::createEof(LastDefaultArgToken.getEndLoc(), Param);
Toks->push_back(DefArgEnd);
// Parse the default argument from its saved token stream.
@@ -482,11 +475,8 @@ void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) {
// Add the 'stop' token.
Token LastExceptionSpecToken = Toks->back();
- Token ExceptionSpecEnd;
- ExceptionSpecEnd.startToken();
- ExceptionSpecEnd.setKind(tok::eof);
- ExceptionSpecEnd.setLocation(LastExceptionSpecToken.getEndLoc());
- ExceptionSpecEnd.setEofData(LM.Method);
+ Token ExceptionSpecEnd =
+ Token::createEof(LastExceptionSpecToken.getEndLoc(), LM.Method);
Toks->push_back(ExceptionSpecEnd);
// Parse the default argument from its saved token stream.
@@ -580,11 +570,7 @@ void Parser::ParseLexedMethodDef(LexedMethod &LM) {
assert(!LM.Toks.empty() && "Empty body!");
Token LastBodyToken = LM.Toks.back();
- Token BodyEnd;
- BodyEnd.startToken();
- BodyEnd.setKind(tok::eof);
- BodyEnd.setLocation(LastBodyToken.getEndLoc());
- BodyEnd.setEofData(LM.D);
+ Token BodyEnd = Token::createEof(LastBodyToken.getEndLoc(), LM.D);
LM.Toks.push_back(BodyEnd);
// Append the current token at the end of the new token stream so that it
// doesn't get lost.
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 8af0ee43bcc33e..24c5549185653b 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -4881,11 +4881,7 @@ void Parser::ParseStructDeclaration(
ParsedAttributes Parser::ParseLexedAttributeTokens(LateParsedAttribute &LPA) {
// Create a fake EOF so that attribute parsing won't go off the end of the
// attribute.
- Token AttrEnd;
- AttrEnd.startToken();
- AttrEnd.setKind(tok::eof);
- AttrEnd.setLocation(Tok.getLocation());
- AttrEnd.setEofData(LPA.Toks.data());
+ Token AttrEnd = Token::createEof(Tok.getLocation(), LPA.Toks.data());
LPA.Toks.push_back(AttrEnd);
// Append the current token at the end of the new token stream so that it
@@ -8334,10 +8330,7 @@ TypeResult Parser::ParseTypeFromString(StringRef TypeStr, StringRef Context,
// Replace the "eod" token with an "eof" token identifying the end of
// the provided string.
Token &EndToken = Tokens.back();
- EndToken.startToken();
- EndToken.setKind(tok::eof);
- EndToken.setLocation(Tok.getLocation());
- EndToken.setEofData(TypeStr.data());
+ EndToken = Token::createEof(Tok.getLocation(), TypeStr.data());
// Add the current token back.
Tokens.push_back(Tok);
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index 8e0613623b9828..aded5d96aa83f4 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -3815,9 +3815,7 @@ void Parser::DiagnoseUnexpectedNamespace(NamedDecl *D) {
// Push '};' onto the token stream to recover.
PP.EnterToken(Tok, /*IsReinject*/ true);
- Tok.startToken();
- Tok.setLocation(PP.getLocForEndOfToken(PrevTokLocation));
- Tok.setKind(tok::semi);
+ Tok = Token::create(tok::semi, PP.getLocForEndOfToken(PrevTokLocation));
PP.EnterToken(Tok, /*IsReinject*/ true);
Tok.setKind(tok::r_brace);
@@ -4383,18 +4381,14 @@ void Parser::ParseOpenMPAttributeArgs(const IdentifierInfo *AttrName,
// If the attribute is named `directive`, we can consume its argument list
// and push the tokens from it into the cached token stream for a new OpenMP
// pragma directive.
- Token OMPBeginTok;
- OMPBeginTok.startToken();
- OMPBeginTok.setKind(tok::annot_attr_openmp);
- OMPBeginTok.setLocation(Tok.getLocation());
+ Token OMPBeginTok =
+ Token::createAnnotation(tok::annot_attr_openmp, Tok.getLocation());
OpenMPTokens.push_back(OMPBeginTok);
ConsumeAndStoreUntil(tok::r_paren, OpenMPTokens, /*StopAtSemi=*/false,
/*ConsumeFinalToken*/ false);
- Token OMPEndTok;
- OMPEndTok.startToken();
- OMPEndTok.setKind(tok::annot_pragma_openmp_end);
- OMPEndTok.setLocation(Tok.getLocation());
+ Token OMPEndTok = Token::createAnnotation(tok::annot_pragma_openmp_end,
+ Tok.getLocation());
OpenMPTokens.push_back(OMPEndTok);
} else {
assert(AttrName->isStr("sequence") &&
@@ -4895,11 +4889,8 @@ void Parser::ParseMicrosoftUuidAttributeArgs(ParsedAttributes &Attrs) {
// ActOnStringLiteral() copies the string data into the literal, so it's
// ok that the Token points to StrBuffer.
Token Toks[1];
- Toks[0].startToken();
- Toks[0].setKind(tok::string_literal);
- Toks[0].setLocation(StartLoc);
+ Toks[0] = Token::create(tok::string_literal, StartLoc, StrBuffer.size());
Toks[0].setLiteralData(StrBuffer.data());
- Toks[0].setLength(StrBuffer.size());
StringLiteral *UuidString =
cast<StringLiteral>(Actions.ActOnUnevaluatedStringLiteral(Toks).get());
ArgExprs.push_back(UuidString);
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 10f70dc64d8a21..d42ffc2ccf0a65 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -341,13 +341,8 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
assert(getLangOpts().Reflection);
if (getLangOpts().Blocks) {
OpToken.setKind(tok::caret);
- Token Caret;
- {
- Caret.startToken();
- Caret.setKind(tok::caret);
- Caret.setLocation(OpToken.getLocation().getLocWithOffset(1));
- Caret.setLength(1);
- }
+ Token Caret = Token::create(
+ tok::caret, OpToken.getLocation().getLocWithOffset(1), 1);
UnconsumeToken(OpToken);
PP.EnterToken(Caret, /*IsReinject=*/true);
return ParseRHSOfBinaryExpression(LHS, MinPrec);
@@ -3202,15 +3197,10 @@ void Parser::injectEmbedTokens() {
Data->BinaryData.size() * 2 - 1);
unsigned I = 0;
for (auto &Byte : Data->BinaryData) {
- Toks[I].startToken();
- Toks[I].setKind(tok::binary_data);
- Toks[I].setLocation(Tok.getLocation());
- Toks[I].setLength(1);
+ Toks[I] = Token::create(tok::binary_data, Tok.getLocation(), 1);
Toks[I].setLiteralData(&Byte);
if (I != ((Data->BinaryData.size() - 1) * 2)) {
- Toks[I + 1].startToken();
- Toks[I + 1].setKind(tok::comma);
- Toks[I + 1].setLocation(Tok.getLocation());
+ Toks[I + 1] = Token::create(tok::comma, Tok.getLocation());
}
I += 2;
}
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index bfdad0d5012825..ea71efc9c9358c 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -3711,11 +3711,7 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
}
// Create a fake EOF to mark end of Toks buffer.
- Token AttrEnd;
- AttrEnd.startToken();
- AttrEnd.setKind(tok::eof);
- AttrEnd.setLocation(Tok.getLocation());
- AttrEnd.setEofData(Toks.data());
+ Token AttrEnd = Token::createEof(Tok.getLocation(), Toks.data());
Toks.push_back(AttrEnd);
// The current token should go after the cached tokens.
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index 7ca08cb2112e67..e63509147fc91b 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -3281,11 +3281,7 @@ void Parser::ParseLexedObjCMethodDefs(LexedMethod &LM, bool parseMethod) {
assert(!LM.Toks.empty() && "ParseLexedObjCMethodDef - Empty body!");
// Store an artificial EOF token to ensure that we don't run off the end of
// the method's body when we come to parse it.
- Token Eof;
- Eof.startToken();
- Eof.setKind(tok::eof);
- Eof.setEofData(MCDecl);
- Eof.setLocation(OrigLoc);
+ Token Eof = Token::createEof(OrigLoc, MCDecl);
LM.Toks.push_back(Eof);
// Append the current token at the end of the new token stream so that it
// doesn't get lost.
diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp
index f58d2c70b7d980..04e7bb5a9b7dd9 100644
--- a/clang/lib/Parse/ParsePragma.cpp
+++ b/clang/lib/Parse/ParsePragma.cpp
@@ -123,12 +123,9 @@ struct PragmaSTDC_FENV_ACCESSHandler : public PragmaHandler {
MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1),
1);
- Toks[0].startToken();
- Toks[0].setKind(tok::annot_pragma_fenv_access);
- Toks[0].setLocation(Tok.getLocation());
- Toks[0].setAnnotationEndLoc(Tok.getLocation());
- Toks[0].setAnnotationValue(reinterpret_cast<void*>(
- static_cast<uintptr_t>(OOS)));
+ Toks[0] = Token::createAnnotation(
+ tok::annot_pragma_fenv_access, Tok.getLocation(),
+ reinterpret_cast<void *>(static_cast<uintptr_t>(OOS)));
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
/*IsReinject=*/false);
}
@@ -147,11 +144,8 @@ struct PragmaSTDC_CX_LIMITED_RANGEHandler : public PragmaHandler {
MutableArrayRef<Token> Toks(
PP.getPreprocessorAllocator().Allocate<Token>(1), 1);
- Toks[0].startToken();
- Toks[0].setKind(tok::annot_pragma_cx_limited_range);
- Toks[0].setLocation(Tok.getLocation());
- Toks[0].setAnnotationEndLoc(Tok.getLocation());
- Toks[0].setAnnotationValue(
+ Toks[0] = Token::createAnnotation(
+ tok::annot_pragma_cx_limited_range, Tok.getLocation(),
reinterpret_cast<void *>(static_cast<uintptr_t>(OOS)));
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
/*IsReinject=*/false);
@@ -353,12 +347,10 @@ struct PragmaMSFenvAccessHandler : public PragmaHandler {
MutableArrayRef<Token> Toks(
PP.getPreprocessorAllocator().Allocate<Token>(1), 1);
- Toks[0].startToken();
- Toks[0].setKind(tok::annot_pragma_fenv_access_ms);
- Toks[0].setLocation(FirstToken.getLocation());
- Toks[0].setAnnotationEndLoc(Tok.getLocation());
- Toks[0].setAnnotationValue(
- reinterpret_cast<void*>(static_cast<uintptr_t>(OOS)));
+ Toks[0] = Token::createAnnotation(
+ tok::annot_pragma_fenv_access_ms,
+ SourceRange(FirstToken.getLocation(), Tok.getLocation()),
+ reinterpret_cast<void *>(static_cast<uintptr_t>(OOS)));
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
/*IsReinject=*/false);
}
@@ -1260,11 +1252,9 @@ bool Parser::HandlePragmaMSInitSeg(StringRef PragmaName,
if (!Section.empty()) {
// Pretend the user wrote the appropriate string literal here.
Token Toks[1];
- Toks[0].startToken();
- Toks[0].setKind(tok::string_literal);
- Toks[0].setLocation(Tok.getLocation());
+ Toks[0] =
+ Token::create(tok::string_literal, Tok.getLocation(), Section.size());
Toks[0].setLiteralData(Section.data());
- Toks[0].setLength(Section.size());
SegmentName =
cast<StringLiteral>(Actions.ActOnStringLiteral(Toks, nullptr).get());
PP.Lex(Tok);
@@ -2190,11 +2180,8 @@ void PragmaGCCVisibilityHandler::HandlePragma(Preprocessor &PP,
}
auto Toks = std::make_unique<Token[]>(1);
- Toks[0].startToken();
- Toks[0].setKind(tok::annot_pragma_vis);
- Toks[0].setLocation(VisLoc);
- Toks[0].setAnnotationEndLoc(EndLoc);
- Toks[0].setAnnotationValue(
+ Toks[0] = Token::createAnnotation(
+ tok::annot_pragma_vis, SourceRange(VisLoc, EndLoc),
const_cast<void *>(static_cast<const void *>(VisType)));
PP.EnterTokenStream(std::move(Toks), 1, /*DisableMacroExpansion=*/true,
/*IsReinject=*/false);
@@ -2334,11 +2321,9 @@ void PragmaPackHandler::HandlePragma(Preprocessor &PP,
MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1),
1);
- Toks[0].startToken();
- Toks[0].setKind(tok::annot_pragma_pack);
- Toks[0].setLocation(PackLoc);
- Toks[0].setAnnotationEndLoc(RParenLoc);
- Toks[0].setAnnotationValue(static_cast<void*>(Info));
+ Toks[0] = Token::createAnnotation(tok::annot_pragma_pack,
+ SourceRange(PackLoc, RParenLoc),
+ static_cast<void *>(Info));
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
/*IsReinject=*/false);
}
@@ -2377,12 +2362,10 @@ void PragmaMSStructHandler::HandlePragma(Preprocessor &PP,
MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1),
1);
- Toks[0].startToken();
- Toks[0].setKind(tok::annot_pragma_msstruct);
- Toks[0].setLocation(MSStructTok.getLocation());
- Toks[0].setAnnotationEndLoc(EndLoc);
- Toks[0].setAnnotationValue(reinterpret_cast<void*>(
- static_cast<uintptr_t>(Kind)));
+ Toks[0] = Token::createAnnotation(
+ tok::annot_pragma_msstruct,
+ SourceRange(MSStructTok.getLocation(), EndLoc),
+ reinterpret_cast<void *>(static_cast<uintptr_t>(Kind)));
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
/*IsReinject=*/false);
}
@@ -2511,12 +2494,9 @@ static void ParseAlignPragma(Preprocessor &PP, Token &FirstTok,
MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1),
1);
- Toks[0].startToken();
- Toks[0].setKind(tok::annot_pragma_align);
- Toks[0].setLocation(FirstTok.getLocation());
- Toks[0].setAnnotationEndLoc(EndLoc);
- Toks[0].setAnnotationValue(reinterpret_cast<void*>(
- static_cast<uintptr_t>(Kind)));
+ Toks[0] = Token::createAnnotation(
+ tok::annot_pragma_align, SourceRange(FirstTok.getLocation(), EndLoc),
+ reinterpret_cast<void *>(static_cast<uintptr_t>(Kind)));
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
/*IsReinject=*/false);
}
@@ -2605,9 +2585,8 @@ void PragmaUnusedHandler::HandlePragma(Preprocessor &PP,
2 * Identifiers.size());
for (unsigned i=0; i != Identifiers.size(); i++) {
Token &pragmaUnusedTok = Toks[2*i], &idTok = Toks[2*i+1];
- pragmaUnusedTok.startToken();
- pragmaUnusedTok.setKind(tok::annot_pragma_unused);
- pragmaUnusedTok.setLocation(UnusedLoc);
+ pragmaUnusedTok =
+ Token::createAnnotation(tok::annot_pragma_unused, UnusedLoc);
idTok = Identifiers[i];
}
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
@@ -2654,10 +2633,9 @@ void PragmaWeakHandler::HandlePragma(Preprocessor &PP,
MutableArrayRef<Token> Toks(
PP.getPreprocessorAllocator().Allocate<Token>(3), 3);
Token &pragmaUnusedTok = Toks[0];
- pragmaUnusedTok.startToken();
- pragmaUnusedTok.setKind(tok::annot_pragma_weakalias);
- pragmaUnusedTok.setLocation(WeakLoc);
- pragmaUnusedTok.setAnnotationEndLoc(AliasName.getLocation());
+ pragmaUnusedTok =
+ Token::createAnnotation(tok::annot_pragma_weakalias,
+ SourceRange(WeakLoc, AliasName.getLocation()));
Toks[1] = WeakName;
Toks[2] = AliasName;
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
@@ -2666,10 +2644,7 @@ void PragmaWeakHandler::HandlePragma(Preprocessor &PP,
MutableArrayRef<Token> Toks(
PP.getPreprocessorAllocator().Allocate<Token>(2), 2);
Token &pragmaUnusedTok = Toks[0];
- pragmaUnusedTok.startToken();
- pragmaUnusedTok.setKind(tok::annot_pragma_weak);
- pragmaUnusedTok.setLocation(WeakLoc);
- pragmaUnusedTok.setAnnotationEndLoc(WeakLoc);
+ pragmaUnusedTok = Token::createAnnotation(tok::annot_pragma_weak, WeakLoc);
Toks[1] = WeakName;
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
/*IsReinject=*/false);
@@ -2711,10 +2686,9 @@ void PragmaRedefineExtnameHandler::HandlePragma(Preprocessor &PP,
MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(3),
3);
Token &pragmaRedefTok = Toks[0];
- pragmaRedefTok.startToken();
- pragmaRedefTok.setKind(tok::annot_pragma_redefine_extname);
- pragmaRedefTok.setLocation(RedefLoc);
- pragmaRedefTok.setAnnotationEndLoc(AliasName.getLocation());
+ pragmaRedefTok =
+ Token::createAnnotation(tok::annot_pragma_redefine_extname,
+ SourceRange(RedefLoc, AliasName.getLocation()));
Toks[1] = RedefName;
Toks[2] = AliasName;
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
@@ -2730,12 +2704,9 @@ void PragmaFPContractHandler::HandlePragma(Preprocessor &PP,
MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1),
1);
- Toks[0].startToken();
- Toks[0].setKind(tok::annot_pragma_fp_contract);
- Toks[0].setLocation(Tok.getLocation());
- Toks[0].setAnnotationEndLoc(Tok.getLocation());
- Toks[0].setAnnotationValue(reinterpret_cast<void*>(
- static_cast<uintptr_t>(OOS)));
+ Toks[0] = Token::createAnnotation(
+ tok::annot_pragma_fp_contract, Tok.getLocation(),
+ reinterpret_cast<void *>(static_cast<uintptr_t>(OOS)));
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
/*IsReinject=*/false);
}
@@ -2793,11 +2764,9 @@ void PragmaOpenCLExtensionHandler::HandlePragma(Preprocessor &PP,
Info->second = State;
MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1),
1);
- Toks[0].startToken();
- Toks[0].setKind(tok::annot_pragma_opencl_extension);
- Toks[0].setLocation(NameLoc);
- Toks[0].setAnnotationValue(static_cast<void*>(Info));
- Toks[0].setAnnotationEndLoc(StateLoc);
+ Toks[0] = Token::createAnnotation(tok::annot_pragma_opencl_extension,
+ SourceRange(NameLoc, StateLoc),
+ static_cast<void *>(Info));
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
/*IsReinject=*/false);
@@ -2826,10 +2795,7 @@ template <tok::TokenKind StartTok, tok::TokenKind EndTok,
void PragmaSupportHandler<StartTok, EndTok, UnexpectedDiag>::HandlePragma(
Preprocessor &PP, PragmaIntroducer Introducer, Token &FirstTok) {
SmallVector<Token, 16> Pragma;
- Token Tok;
- Tok.startToken();
- Tok.setKind(StartTok);
- Tok.setLocation(Introducer.Loc);
+ Token Tok = Token::createAnnotation(StartTok, Introducer.Loc);
while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof)) {
Pragma.push_back(Tok);
@@ -2848,9 +2814,7 @@ void PragmaSupportHandler<StartTok, EndTok, UnexpectedDiag>::HandlePragma(
}
}
SourceLocation EodLoc = Tok.getLocation();
- Tok.startToken();
- Tok.setKind(EndTok);
- Tok.setLocation(EodLoc);
+ Tok = Token::createAnnotation(EndTok, EodLoc);
Pragma.push_back(Tok);
auto Toks = std::make_unique<Token[]>(Pragma.size());
@@ -2947,12 +2911,9 @@ void PragmaMSPointersToMembers::HandlePragma(Preprocessor &PP,
return;
}
- Token AnnotTok;
- AnnotTok.startToken();
- AnnotTok.setKind(tok::annot_pragma_ms_pointers_to_members);
- AnnotTok.setLocation(PointersToMembersLoc);
- AnnotTok.setAnnotationEndLoc(EndLoc);
- AnnotTok.setAnnotationValue(
+ Token AnnotTok = Token::createAnnotation(
+ tok::annot_pragma_ms_pointers_to_members,
+ SourceRange(PointersToMembersLoc, EndLoc),
reinterpret_cast<void *>(static_cast<uintptr_t>(RepresentationMethod)));
PP.EnterToken(AnnotTok, /*IsReinject=*/true);
}
@@ -3039,13 +3000,10 @@ void PragmaMSVtorDisp::HandlePragma(Preprocessor &PP,
}
// Enter the annotation.
- Token AnnotTok;
- AnnotTok.startToken();
- AnnotTok.setKind(tok::annot_pragma_ms_vtordisp);
- AnnotTok.setLocation(VtorDispLoc);
- AnnotTok.setAnnotationEndLoc(EndLoc);
- AnnotTok.setAnnotationValue(reinterpret_cast<void *>(
- static_cast<uintptr_t>((Action << 16) | (Value & 0xFFFF))));
+ Token AnnotTok = Token::createAnnotation(
+ tok::annot_pragma_ms_vtordisp, SourceRange(VtorDispLoc, EndLoc),
+ reinterpret_cast<void *>(
+ static_cast<uintptr_t>((Action << 16) | (Value & 0xFFFF))));
PP.EnterToken(AnnotTok, /*IsReinject=*/false);
}
@@ -3053,13 +3011,9 @@ void PragmaMSVtorDisp::HandlePragma(Preprocessor &PP,
/// an annotation token.
void PragmaMSPragma::HandlePragma(Preprocessor &PP,
PragmaIntroducer Introducer, Token &Tok) {
- Token EoF, AnnotTok;
- EoF.startToken();
- EoF.setKind(tok::eof);
- AnnotTok.startToken();
- AnnotTok.setKind(tok::annot_pragma_ms_pragma);
- AnnotTok.setLocation(Tok.getLocation());
- AnnotTok.setAnnotationEndLoc(Tok.getLocation());
+ Token EoF = Token::createEof();
+ Token AnnotTok =
+ Token::createAnnotation(tok::annot_pragma_ms_pragma, Tok.getLocation());
SmallVector<Token, 8> TokenVector;
// Suck up all of the tokens before the eod.
for (; Tok.isNot(tok::eod); PP.Lex(Tok)) {
@@ -3197,14 +3151,12 @@ void PragmaFloatControlHandler::HandlePragma(Preprocessor &PP,
// Enter the annotation.
auto TokenArray = std::make_unique<Token[]>(1);
- TokenArray[0].startToken();
- TokenArray[0].setKind(tok::annot_pragma_float_control);
- TokenArray[0].setLocation(FloatControlLoc);
- TokenArray[0].setAnnotationEndLoc(EndLoc);
// Create an encoding of Action and Value by shifting the Action into
// the high 16 bits then union with the Kind.
- TokenArray[0].setAnnotationValue(reinterpret_cast<void *>(
- static_cast<uintptr_t>((Action << 16) | (Kind & 0xFFFF))));
+ TokenArray[0] = Token::createAnnotation(
+ tok::annot_pragma_float_control, SourceRange(FloatControlLoc, EndLoc),
+ reinterpret_cast<void *>(
+ static_cast<uintptr_t>((Action << 16) | (Kind & 0xFFFF))));
PP.EnterTokenStream(std::move(TokenArray), 1,
/*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
@@ -3542,12 +3494,9 @@ void PragmaFPHandler::HandlePragma(Preprocessor &PP,
return;
}
- Token FPTok;
- FPTok.startToken();
- FPTok.setKind(tok::annot_pragma_fp);
- FPTok.setLocation(PragmaName.getLocation());
- FPTok.setAnnotationEndLoc(PragmaName.getLocation());
- FPTok.setAnnotationValue(reinterpret_cast<void *>(AnnotValue));
+ Token FPTok =
+ Token::createAnnotation(tok::annot_pragma_fp, PragmaName.getLocation(),
+ reinterpret_cast<void *>(AnnotValue));
TokenList.push_back(FPTok);
auto TokenArray = std::make_unique<Token[]>(TokenList.size());
@@ -3601,11 +3550,8 @@ void PragmaSTDC_FENV_ROUNDHandler::HandlePragma(Preprocessor &PP,
MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1),
1);
- Toks[0].startToken();
- Toks[0].setKind(tok::annot_pragma_fenv_round);
- Toks[0].setLocation(Tok.getLocation());
- Toks[0].setAnnotationEndLoc(Tok.getLocation());
- Toks[0].setAnnotationValue(
+ Toks[0] = Token::createAnnotation(
+ tok::annot_pragma_fenv_round, Tok.getLocation(),
reinterpret_cast<void *>(static_cast<uintptr_t>(RM)));
PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
/*IsReinject=*/false);
@@ -3667,10 +3613,7 @@ static bool ParseLoopHintValue(Preprocessor &PP, Token &Tok, Token PragmaName,
PP.Lex(Tok);
}
- Token EOFTok;
- EOFTok.startToken();
- EOFTok.setKind(tok::eof);
- EOFTok.setLocation(Tok.getLocation());
+ Token EOFTok = Token::createEof(Tok.getLocation());
ValueList.push_back(EOFTok); // Terminates expression for parsing.
markAsReinjectedForRelexing(ValueList);
@@ -3781,12 +3724,10 @@ void PragmaLoopHintHandler::HandlePragma(Preprocessor &PP,
return;
// Generate the loop hint token.
- Token LoopHintTok;
- LoopHintTok.startToken();
- LoopHintTok.setKind(tok::annot_pragma_loop_hint);
- LoopHintTok.setLocation(Introducer.Loc);
- LoopHintTok.setAnnotationEndLoc(PragmaName.getLocation());
- LoopHintTok.setAnnotationValue(static_cast<void *>(Info));
+ Token LoopHintTok = Token::createAnnotation(
+ tok::annot_pragma_loop_hint,
+ SourceRange(Introducer.Loc, PragmaName.getLocation()),
+ static_cast<void *>(Info));
TokenList.push_back(LoopHintTok);
}
@@ -3869,11 +3810,10 @@ void PragmaUnrollHintHandler::HandlePragma(Preprocessor &PP,
// Generate the hint token.
auto TokenArray = std::make_unique<Token[]>(1);
- TokenArray[0].startToken();
- TokenArray[0].setKind(tok::annot_pragma_loop_hint);
- TokenArray[0].setLocation(Introducer.Loc);
- TokenArray[0].setAnnotationEndLoc(PragmaName.getLocation());
- TokenArray[0].setAnnotationValue(static_cast<void *>(Info));
+ TokenArray[0] = Token::createAnnotation(
+ tok::annot_pragma_loop_hint,
+ SourceRange(Introducer.Loc, PragmaName.getLocation()),
+ static_cast<void *>(Info));
PP.EnterTokenStream(std::move(TokenArray), 1,
/*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
@@ -4166,10 +4106,7 @@ void PragmaAttributeHandler::HandlePragma(Preprocessor &PP,
PP.Lex(Tok);
// Terminate the attribute for parsing.
- Token EOFTok;
- EOFTok.startToken();
- EOFTok.setKind(tok::eof);
- EOFTok.setLocation(EndLoc);
+ Token EOFTok = Token::createEof(EndLoc);
AttributeTokens.push_back(EOFTok);
markAsReinjectedForRelexing(AttributeTokens);
@@ -4183,11 +4120,9 @@ void PragmaAttributeHandler::HandlePragma(Preprocessor &PP,
// Generate the annotated pragma token.
auto TokenArray = std::make_unique<Token[]>(1);
- TokenArray[0].startToken();
- TokenArray[0].setKind(tok::annot_pragma_attribute);
- TokenArray[0].setLocation(FirstToken.getLocation());
- TokenArray[0].setAnnotationEndLoc(FirstToken.getLocation());
- TokenArray[0].setAnnotationValue(static_cast<void *>(Info));
+ TokenArray[0] = Token::createAnnotation(tok::annot_pragma_attribute,
+ FirstToken.getLocation(),
+ static_cast<void *>(Info));
PP.EnterTokenStream(std::move(TokenArray), 1,
/*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
@@ -4255,11 +4190,7 @@ void PragmaMaxTokensTotalHandler::HandlePragma(Preprocessor &PP,
static void zOSPragmaHandlerHelper(Preprocessor &PP, Token &Tok,
tok::TokenKind TokKind) {
- Token AnnotTok;
- AnnotTok.startToken();
- AnnotTok.setKind(TokKind);
- AnnotTok.setLocation(Tok.getLocation());
- AnnotTok.setAnnotationEndLoc(Tok.getLocation());
+ Token AnnotTok = Token::createAnnotation(TokKind, Tok.getLocation());
SmallVector<Token, 8> TokenVector;
// Suck up all of the tokens before the eod.
for (; Tok.isNot(tok::eod); PP.Lex(Tok)) {
@@ -4267,10 +4198,7 @@ static void zOSPragmaHandlerHelper(Preprocessor &PP, Token &Tok,
AnnotTok.setAnnotationEndLoc(Tok.getLocation());
}
// Add a sentinel EoF token to the end of the list.
- Token EoF;
- EoF.startToken();
- EoF.setKind(tok::eof);
- EoF.setLocation(Tok.getLocation());
+ Token EoF = Token::createEof(Tok.getLocation());
TokenVector.push_back(EoF);
// We must allocate this array with new because EnterTokenStream is going to
// delete it later.
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index e35666b5d6420a..c5d6a17fbfcd34 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -77,8 +77,7 @@ Parser::Parser(Preprocessor &pp, Sema &actions, bool skipFunctionBodies)
ParsingGenericAssociationType(false), InMessageExpression(false),
ParsingInObjCContainer(false), TemplateParameterDepth(0) {
SkipFunctionBodies = pp.isCodeCompletionEnabled() || skipFunctionBodies;
- Tok.startToken();
- Tok.setKind(tok::eof);
+ Tok = Token::createEof();
Actions.CurScope = nullptr;
NumCachedScopes = 0;
CurParsedObjCImpl = nullptr;
diff --git a/clang/lib/Rewrite/TokenRewriter.cpp b/clang/lib/Rewrite/TokenRewriter.cpp
index b1f4bd251580a2..be3cacee47b7fb 100644
--- a/clang/lib/Rewrite/TokenRewriter.cpp
+++ b/clang/lib/Rewrite/TokenRewriter.cpp
@@ -85,15 +85,11 @@ TokenRewriter::AddTokenBefore(token_iterator I, const char *Val) {
// Plop the string into the scratch buffer, then create a token for this
// string.
- Token Tok;
- Tok.startToken();
const char *Spelling;
- Tok.setLocation(ScratchBuf->getToken(Val, Len, Spelling));
- Tok.setLength(Len);
-
// TODO: Form a whole lexer around this and relex the token! For now, just
// set kind to tok::unknown.
- Tok.setKind(tok::unknown);
+ Token Tok = Token::create(tok::unknown,
+ ScratchBuf->getToken(Val, Len, Spelling), Len);
return AddToken(Tok, RemapIterator(I));
}
>From 2aac9fc4eec2281ae8660da55a2463afd37092cd Mon Sep 17 00:00:00 2001
From: Henrich Lauko <xlauko at mail.muni.cz>
Date: Tue, 6 Oct 2026 23:02:33 +0200
Subject: [PATCH 24/46] [CIR][CodeGen][NFC] Share the bit-field and vbase
layout ABI predicates (#227258)
Deduplicates `isDiscreteBitFieldABI` and `isOverlappingVBaseABI` between CIR and
classic CodeGen into `RecordLayoutUtils.h`, as free functions taking the
`ASTContext`.
Assisted-by: Claude Code (Claude Fable 5.1).
---
.../clang/CodeGenUtils/RecordLayoutUtils.h | 12 ++++++++
.../CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp | 22 ++------------
clang/lib/CodeGen/CGRecordLayoutBuilder.cpp | 30 ++++---------------
clang/lib/CodeGenUtils/RecordLayoutUtils.cpp | 9 ++++++
4 files changed, 30 insertions(+), 43 deletions(-)
diff --git a/clang/include/clang/CodeGenUtils/RecordLayoutUtils.h b/clang/include/clang/CodeGenUtils/RecordLayoutUtils.h
index 77f14fee5a3b4c..2c1ee3c271a3d1 100644
--- a/clang/include/clang/CodeGenUtils/RecordLayoutUtils.h
+++ b/clang/include/clang/CodeGenUtils/RecordLayoutUtils.h
@@ -18,6 +18,12 @@
namespace clang::CodeGenUtils {
+/// The Microsoft bitfield layout rule allocates discrete storage units of the
+/// field's formal type and only combines adjacent fields of the same formal
+/// type. We want to emit a layout with these discrete storage units instead
+/// of combining them into a continuous run.
+bool isDiscreteBitFieldABI(const ASTContext &Ctx, const RecordDecl *RD);
+
/// Return true iff the field is "empty", that is, either a zero-width
/// bit-field or an \ref isEmptyRecordForLayout.
bool isEmptyFieldForLayout(const ASTContext &Ctx, const FieldDecl *FD);
@@ -29,6 +35,12 @@ bool isEmptyFieldForLayout(const ASTContext &Ctx, const FieldDecl *FD);
/// isEmptyRecord.
bool isEmptyRecordForLayout(const ASTContext &Ctx, QualType T);
+/// The Itanium base layout rule allows virtual bases to overlap other bases,
+/// which complicates layout in specific ways.
+///
+/// Note specifically that the ms_struct attribute doesn't change this.
+bool isOverlappingVBaseABI(const ASTContext &Ctx);
+
} // namespace clang::CodeGenUtils
#endif // LLVM_CLANG_CODEGENUTILS_RECORDLAYOUTUTILS_H
diff --git a/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp b/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
index 8617ba5561e415..982a0923a50674 100644
--- a/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
@@ -134,27 +134,10 @@ struct CIRRecordLowering final {
/// Helper function to check if the target machine is BigEndian.
bool isBigEndian() const { return astContext.getTargetInfo().isBigEndian(); }
- // The Itanium base layout rule allows virtual bases to overlap
- // other bases, which complicates layout in specific ways.
- //
- // Note specifically that the ms_struct attribute doesn't change this.
- bool isOverlappingVBaseABI() {
- return !astContext.getTargetInfo().getCXXABI().isMicrosoft();
- }
// Recursively searches all of the bases to find out if a vbase is
// not the primary vbase of some base class.
bool hasOwnStorage(const CXXRecordDecl *decl, const CXXRecordDecl *query);
- /// The Microsoft bitfield layout rule allocates discrete storage
- /// units of the field's formal type and only combines adjacent
- /// fields of the same formal type. We want to emit a layout with
- /// these discrete storage units instead of combining them into a
- /// continuous run.
- bool isDiscreteBitFieldABI() {
- return astContext.getTargetInfo().getCXXABI().isMicrosoft() ||
- recordDecl->isMsStruct(astContext);
- }
-
CharUnits bitsToCharUnits(uint64_t bitOffset) {
return astContext.toCharUnitsFromBits(bitOffset);
}
@@ -446,7 +429,7 @@ void CIRRecordLowering::fillOutputFields() {
RecordDecl::field_iterator
CIRRecordLowering::accumulateBitFields(RecordDecl::field_iterator field,
RecordDecl::field_iterator fieldEnd) {
- if (isDiscreteBitFieldABI()) {
+ if (CodeGenUtils::isDiscreteBitFieldABI(astContext, recordDecl)) {
// run stores the first element of the current run of bitfields. fieldEnd is
// used as a special value to note that we don't have a current run. A
// bitfield run is a contiguous collection of bitfields that can be stored
@@ -1293,7 +1276,8 @@ void CIRRecordLowering::accumulateVBases() {
CharUnits offset = astRecordLayout.getVBaseClassOffset(baseDecl);
// If the vbase is a primary virtual base of some base, then it doesn't
// get its own storage location but instead lives inside of that base.
- if (isOverlappingVBaseABI() && astContext.isNearlyEmpty(baseDecl) &&
+ if (CodeGenUtils::isOverlappingVBaseABI(astContext) &&
+ astContext.isNearlyEmpty(baseDecl) &&
!hasOwnStorage(cxxRecordDecl, baseDecl)) {
members.push_back(MemberInfo(offset, MemberInfo::InfoKind::VBase, nullptr,
cir::RecordMemberKind::Data, baseDecl));
diff --git a/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp b/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
index 14c4b969e141cc..9b2f6426cfc84e 100644
--- a/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
+++ b/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
@@ -101,27 +101,9 @@ struct CGRecordLowering {
return MemberInfo(Offset, MemberInfo::Field, Data);
}
- /// The Microsoft bitfield layout rule allocates discrete storage
- /// units of the field's formal type and only combines adjacent
- /// fields of the same formal type. We want to emit a layout with
- /// these discrete storage units instead of combining them into a
- /// continuous run.
- bool isDiscreteBitFieldABI() const {
- return Context.getTargetInfo().getCXXABI().isMicrosoft() ||
- D->isMsStruct(Context);
- }
-
/// Helper function to check if the target machine is BigEndian.
bool isBE() const { return Context.getTargetInfo().isBigEndian(); }
- /// The Itanium base layout rule allows virtual bases to overlap
- /// other bases, which complicates layout in specific ways.
- ///
- /// Note specifically that the ms_struct attribute doesn't change this.
- bool isOverlappingVBaseABI() const {
- return !Context.getTargetInfo().getCXXABI().isMicrosoft();
- }
-
/// Wraps llvm::Type::getIntNTy with some implicit arguments.
llvm::Type *getIntNType(uint64_t NumBits) const {
unsigned AlignedBits = llvm::alignTo(NumBits, Context.getCharWidth());
@@ -144,7 +126,8 @@ struct CGRecordLowering {
llvm::Type *getStorageType(const FieldDecl *FD) const {
llvm::Type *Type = Types.ConvertTypeForMem(FD->getType());
if (!FD->isBitField()) return Type;
- if (isDiscreteBitFieldABI()) return Type;
+ if (CodeGenUtils::isDiscreteBitFieldABI(Context, D))
+ return Type;
return getIntNType(std::min(FD->getBitWidthValue(),
(unsigned)Context.toBits(getSize(Type))));
}
@@ -407,7 +390,7 @@ RecordDecl::field_iterator
CGRecordLowering::accumulateBitFields(bool isNonVirtualBaseType,
RecordDecl::field_iterator Field,
RecordDecl::field_iterator FieldEnd) {
- if (isDiscreteBitFieldABI()) {
+ if (CodeGenUtils::isDiscreteBitFieldABI(Context, D)) {
// Run stores the first element of the current run of bitfields. FieldEnd is
// used as a special value to note that we don't have a current run. A
// bitfield run is a contiguous collection of bitfields that can be stored
@@ -875,7 +858,7 @@ CGRecordLowering::calculateTailClippingOffset(bool isNonVirtualBaseType) const {
// smaller than the nvsize. Here we check to see if such a base is placed
// before the nvsize and set the scissor offset to that, instead of the
// nvsize.
- if (!isNonVirtualBaseType && isOverlappingVBaseABI())
+ if (!isNonVirtualBaseType && CodeGenUtils::isOverlappingVBaseABI(Context))
for (const auto &Base : RD->vbases()) {
const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl();
if (CodeGenUtils::isEmptyRecordForLayout(Context, Base.getType()))
@@ -899,9 +882,8 @@ void CGRecordLowering::accumulateVBases() {
CharUnits Offset = Layout.getVBaseClassOffset(BaseDecl);
// If the vbase is a primary virtual base of some base, then it doesn't
// get its own storage location but instead lives inside of that base.
- if (isOverlappingVBaseABI() &&
- Context.isNearlyEmpty(BaseDecl) &&
- !hasOwnStorage(RD, BaseDecl)) {
+ if (CodeGenUtils::isOverlappingVBaseABI(Context) &&
+ Context.isNearlyEmpty(BaseDecl) && !hasOwnStorage(RD, BaseDecl)) {
Members.push_back(MemberInfo(Offset, MemberInfo::VBase, nullptr,
BaseDecl));
continue;
diff --git a/clang/lib/CodeGenUtils/RecordLayoutUtils.cpp b/clang/lib/CodeGenUtils/RecordLayoutUtils.cpp
index 3b8d433dc8759b..b41fb61491377f 100644
--- a/clang/lib/CodeGenUtils/RecordLayoutUtils.cpp
+++ b/clang/lib/CodeGenUtils/RecordLayoutUtils.cpp
@@ -7,9 +7,14 @@
//===----------------------------------------------------------------------===//
#include "clang/CodeGenUtils/RecordLayoutUtils.h"
+#include "clang/Basic/TargetInfo.h"
namespace clang::CodeGenUtils {
+bool isDiscreteBitFieldABI(const ASTContext &Ctx, const RecordDecl *RD) {
+ return Ctx.getTargetInfo().getCXXABI().isMicrosoft() || RD->isMsStruct(Ctx);
+}
+
bool isEmptyFieldForLayout(const ASTContext &Ctx, const FieldDecl *FD) {
if (FD->isZeroLengthBitField())
return true;
@@ -42,4 +47,8 @@ bool isEmptyRecordForLayout(const ASTContext &Ctx, QualType T) {
return true;
}
+bool isOverlappingVBaseABI(const ASTContext &Ctx) {
+ return !Ctx.getTargetInfo().getCXXABI().isMicrosoft();
+}
+
} // namespace clang::CodeGenUtils
>From b8eef82a624daff348a28b93e5320c986bde5f44 Mon Sep 17 00:00:00 2001
From: Domenic Nutile <domenic.nutile at gmail.com>
Date: Tue, 6 Oct 2026 17:04:45 -0400
Subject: [PATCH 25/46] [AMDGPU][SROA] Expand cast chain handling to floating
point types (#228113)
CreateBitPreservingCastChain does not produce inttoptr or ptrtoint for
floating-point types. When promoting structs like { float, float } to <2
x float>, this can lead to pointers being bitcast directory to <2 x
float> which is invalid. This change expands the use of the intermediate
to these cases
---
llvm/lib/IR/IRBuilder.cpp | 8 +++-
.../AMDGPU/promote-alloca-loadstores.ll | 48 +++++++++++++++++++
2 files changed, 54 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 6653f8f21b6935..07125506e6094a 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -140,19 +140,23 @@ Value *IRBuilderBase::CreateBitPreservingCastChain(const DataLayout &DL,
};
// See if we need inttoptr for this type pair. May require additional bitcast.
- if (OldTy->isIntOrIntVectorTy() && NewTy->isPtrOrPtrVectorTy()) {
+ bool OldIsIntOrFP = OldTy->isIntOrIntVectorTy() || OldTy->isFPOrFPVectorTy();
+ if (OldIsIntOrFP && NewTy->isPtrOrPtrVectorTy()) {
// Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8*
// Expand i128 to <2 x i8*> --> i128 to <2 x i64> to <2 x i8*>
// Expand <4 x i32> to <2 x i8*> --> <4 x i32> to <2 x i64> to <2 x i8*>
+ // Expand <2 x float> to i8* --> <2 x float> to i64 to i8*
// Directly handle i64 to i8*
return CreateIntToPtr(CreateBitCastLike(V, DL.getIntPtrType(NewTy)), NewTy);
}
// See if we need ptrtoint for this type pair. May require additional bitcast.
- if (OldTy->isPtrOrPtrVectorTy() && NewTy->isIntOrIntVectorTy()) {
+ bool NewIsIntOrFP = NewTy->isIntOrIntVectorTy() || NewTy->isFPOrFPVectorTy();
+ if (OldTy->isPtrOrPtrVectorTy() && NewIsIntOrFP) {
// Expand <2 x i8*> to i128 --> <2 x i8*> to <2 x i64> to i128
// Expand i8* to <2 x i32> --> i8* to i64 to <2 x i32>
// Expand <2 x i8*> to <4 x i32> --> <2 x i8*> to <2 x i64> to <4 x i32>
+ // Expand i8* to <2 x float> --> i8* to i64 to <2 x float>
// Expand i8* to i64 --> i8* to i64 to i64
return CreateBitCastLike(CreatePtrToInt(V, DL.getIntPtrType(OldTy)), NewTy);
}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-loadstores.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-loadstores.ll
index 0773a60453de6e..228d4ef1d771f5 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-loadstores.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-loadstores.ll
@@ -259,3 +259,51 @@ entry:
%tmp = load ptr addrspace(3), ptr addrspace(5) %alloca, align 8
ret ptr addrspace(3) %tmp
}
+
+define <2 x float> @alloca_store_ptr_load_fvec(ptr %arg) {
+; CHECK-LABEL: define <2 x float> @alloca_store_ptr_load_fvec
+; CHECK-SAME: (ptr [[ARG:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[ALLOCA:%.*]] = freeze <2 x float> poison
+; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x float>
+; CHECK-NEXT: ret <2 x float> [[TMP1]]
+;
+entry:
+ %alloca = alloca [2 x float], align 8, addrspace(5)
+ store ptr %arg, ptr addrspace(5) %alloca, align 8
+ %tmp = load <2 x float>, ptr addrspace(5) %alloca, align 8
+ ret <2 x float> %tmp
+}
+
+define ptr @alloca_store_fvec_load_ptr(<2 x float> %arg) {
+; CHECK-LABEL: define ptr @alloca_store_fvec_load_ptr
+; CHECK-SAME: (<2 x float> [[ARG:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[ALLOCA:%.*]] = freeze <2 x float> poison
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x float> [[ARG]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT: ret ptr [[TMP1]]
+;
+entry:
+ %alloca = alloca [2 x float], align 8, addrspace(5)
+ store <2 x float> %arg, ptr addrspace(5) %alloca, align 8
+ %tmp = load ptr, ptr addrspace(5) %alloca, align 8
+ ret ptr %tmp
+}
+
+define <2 x ptr> @alloca_store_fvec_load_ptrvec(<4 x float> %arg) {
+; CHECK-LABEL: define <2 x ptr> @alloca_store_fvec_load_ptrvec
+; CHECK-SAME: (<4 x float> [[ARG:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[ALLOCA:%.*]] = freeze <4 x float> poison
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x float> [[ARG]] to <2 x i64>
+; CHECK-NEXT: [[TMP1:%.*]] = inttoptr <2 x i64> [[TMP0]] to <2 x ptr>
+; CHECK-NEXT: ret <2 x ptr> [[TMP1]]
+;
+entry:
+ %alloca = alloca [4 x float], align 16, addrspace(5)
+ store <4 x float> %arg, ptr addrspace(5) %alloca, align 16
+ %tmp = load <2 x ptr>, ptr addrspace(5) %alloca, align 16
+ ret <2 x ptr> %tmp
+}
>From 2e11f7391d434cde5106208db597c1e2c9b6fdd5 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 6 Oct 2026 22:20:19 +0100
Subject: [PATCH 26/46] [llubi] Support the oracle form of
llvm.speculative.load. (#227095)
Add support for llvm.speculative.load with oracle functions.
This is split into 2 steps:
1. enterCall checks if CB is a speculative load in oracle form. If so,
adjust Callee to the oracle functions and drop the non-oracle arguments.
2. returnFromCallee also hecks if the current PC is a speculative load
with oracle. If so, it calls callSpeculativeLoadIntrinsic before
returning.
Follow-up to https://github.com/llvm/llvm-project/pull/226839
PR: https://github.com/llvm/llvm-project/pull/227095
---
.../test/tools/llubi/intr_speculative_load.ll | 22 ------
.../llubi/intr_speculative_load_oracle.ll | 70 +++++++++++++++++++
.../llubi/intr_speculative_load_oracle_ub.ll | 69 ++++++++++++++++++
.../tools/llubi/intr_speculative_load_ub.ll | 33 +--------
llvm/tools/llubi/lib/Interpreter.cpp | 37 +++++++---
5 files changed, 170 insertions(+), 61 deletions(-)
create mode 100644 llvm/test/tools/llubi/intr_speculative_load_oracle.ll
create mode 100644 llvm/test/tools/llubi/intr_speculative_load_oracle_ub.ll
diff --git a/llvm/test/tools/llubi/intr_speculative_load.ll b/llvm/test/tools/llubi/intr_speculative_load.ll
index 8e927f5b49038d..4b32766e10c3ec 100644
--- a/llvm/test/tools/llubi/intr_speculative_load.ll
+++ b/llvm/test/tools/llubi/intr_speculative_load.ll
@@ -1,19 +1,9 @@
; RUN: llubi --verbose --entry-function=first_bytes < %s 2>&1 | FileCheck %s --check-prefix=FIRST
; RUN: llubi --verbose --entry-function=last_bytes < %s 2>&1 | FileCheck %s --check-prefix=LAST
; RUN: llubi --verbose --entry-function=past_end < %s 2>&1 | FileCheck %s --check-prefix=PAST-END
-; RUN: not llubi --verbose --entry-function=oracle_load < %s 2>&1 | FileCheck %s --check-prefix=ORACLE
@a = global [6 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5]
-; Returns the number of bytes from %p to %end, clamped to 16.
-define i64 @oracle(ptr %p, ptr %end) memory(none) nounwind nosync willreturn {
- %p.int = ptrtoaddr ptr %p to i64
- %end.int = ptrtoaddr ptr %end to i64
- %diff = sub i64 %end.int, %p.int
- %n = call i64 @llvm.umin.i64(i64 %diff, i64 16)
- ret i64 %n
-}
-
define void @first_bytes() {
; FIRST: Entering function: first_bytes
; FIRST-NEXT: %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr @a, i1 false, i64 8) => { i32 0, i32 1, poison, poison }
@@ -42,15 +32,3 @@ define void @past_end() {
%r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr %p, i1 false, i64 8)
ret void
}
-
-define void @oracle_load() {
-; ORACLE: Entering function: oracle_load
-; ORACLE-NEXT: %p = getelementptr i32, ptr @a, i64 4 => ptr 0x20 [@a + 16]
-; ORACLE-NEXT: %end = getelementptr i32, ptr @a, i64 6 => ptr 0x28 [@a + 24]
-; ORACLE-NEXT: Unrecognized instruction: %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr %p, i1 false, ptr @oracle, ptr %p, ptr %end)
-; ORACLE-NEXT: error: Execution of function 'oracle_load' failed.
- %p = getelementptr i32, ptr @a, i64 4
- %end = getelementptr i32, ptr @a, i64 6
- %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr %p, i1 false, ptr @oracle, ptr %p, ptr %end)
- ret void
-}
diff --git a/llvm/test/tools/llubi/intr_speculative_load_oracle.ll b/llvm/test/tools/llubi/intr_speculative_load_oracle.ll
new file mode 100644
index 00000000000000..dc413612491bfd
--- /dev/null
+++ b/llvm/test/tools/llubi/intr_speculative_load_oracle.ll
@@ -0,0 +1,70 @@
+; RUN: llubi --verbose --entry-function=oracle_load < %s 2>&1 | FileCheck %s --check-prefix=ORACLE
+; RUN: llubi --verbose --entry-function=nested_oracle_load < %s 2>&1 | FileCheck %s --check-prefix=NESTED
+
+ at a = global [6 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5]
+
+; Returns the number of bytes from %p to %end, clamped to 16.
+define i64 @oracle(ptr %p, ptr %end) memory(none) nounwind nosync willreturn {
+ %p.int = ptrtoaddr ptr %p to i64
+ %end.int = ptrtoaddr ptr %end to i64
+ %diff = sub i64 %end.int, %p.int
+ %n = call i64 @llvm.umin.i64(i64 %diff, i64 16)
+ ret i64 %n
+}
+
+; Returns the second element at %p plus 3, loaded via an oracle-form
+; llvm.speculative.load.
+define i64 @oracle_nested(ptr %p) memory(argmem: read) nounwind nosync willreturn {
+ %v = call <2 x i32> (ptr, i1, ...) @llvm.speculative.load.v2i32.p0(ptr %p, i1 false, ptr @oracle_const, i64 8)
+ %e = extractelement <2 x i32> %v, i64 1
+ %z = zext i32 %e to i64
+ %n = add i64 %z, 3
+ ret i64 %n
+}
+
+define i64 @oracle_const(i64 %n) memory(none) nounwind nosync willreturn {
+ ret i64 %n
+}
+
+define void @oracle_load() {
+; ORACLE: Entering function: oracle_load
+; ORACLE-NEXT: %p = getelementptr i32, ptr @a, i64 4 => ptr 0x20 [@a + 16]
+; ORACLE-NEXT: %end = getelementptr i32, ptr @a, i64 6 => ptr 0x28 [@a + 24]
+; ORACLE-NEXT: Entering function: oracle
+; ORACLE-NEXT: ptr %p = ptr 0x20 [@a + 16]
+; ORACLE-NEXT: ptr %end = ptr 0x28 [@a + 24]
+; ORACLE-NEXT: %p.int = ptrtoaddr ptr %p to i64 => i64 32
+; ORACLE-NEXT: %end.int = ptrtoaddr ptr %end to i64 => i64 40
+; ORACLE-NEXT: %diff = sub i64 %end.int, %p.int => i64 8
+; ORACLE-NEXT: %n = call i64 @llvm.umin.i64(i64 %diff, i64 16) => i64 8
+; ORACLE-NEXT: ret i64 %n
+; ORACLE-NEXT: Exiting function: oracle
+; ORACLE-NEXT: %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr %p, i1 false, ptr @oracle, ptr %p, ptr %end) => { i32 4, i32 5, poison, poison }
+; ORACLE-NEXT: ret void
+; ORACLE-NEXT: Exiting function: oracle_load
+ %p = getelementptr i32, ptr @a, i64 4
+ %end = getelementptr i32, ptr @a, i64 6
+ %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr %p, i1 false, ptr @oracle, ptr %p, ptr %end)
+ ret void
+}
+
+define void @nested_oracle_load() {
+; NESTED: Entering function: nested_oracle_load
+; NESTED-NEXT: Entering function: oracle_nested
+; NESTED-NEXT: ptr %p = ptr 0x10 [@a]
+; NESTED-NEXT: Entering function: oracle_const
+; NESTED-NEXT: i64 %n = i64 8
+; NESTED-NEXT: ret i64 %n
+; NESTED-NEXT: Exiting function: oracle_const
+; NESTED-NEXT: %v = call <2 x i32> (ptr, i1, ...) @llvm.speculative.load.v2i32.p0(ptr %p, i1 false, ptr @oracle_const, i64 8) => { i32 0, i32 1 }
+; NESTED-NEXT: %e = extractelement <2 x i32> %v, i64 1 => i32 1
+; NESTED-NEXT: %z = zext i32 %e to i64 => i64 1
+; NESTED-NEXT: %n = add i64 %z, 3 => i64 4
+; NESTED-NEXT: ret i64 %n
+; NESTED-NEXT: Exiting function: oracle_nested
+; NESTED-NEXT: %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr @a, i1 true, ptr @oracle_nested, ptr @a) => { poison, poison, poison, i32 3 }
+; NESTED-NEXT: ret void
+; NESTED-NEXT: Exiting function: nested_oracle_load
+ %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr @a, i1 true, ptr @oracle_nested, ptr @a)
+ ret void
+}
diff --git a/llvm/test/tools/llubi/intr_speculative_load_oracle_ub.ll b/llvm/test/tools/llubi/intr_speculative_load_oracle_ub.ll
new file mode 100644
index 00000000000000..5308795b653290
--- /dev/null
+++ b/llvm/test/tools/llubi/intr_speculative_load_oracle_ub.ll
@@ -0,0 +1,69 @@
+; RUN: not llubi --verbose --entry-function=oracle_out_of_bounds < %s 2>&1 | FileCheck %s --check-prefix=ORACLE-OOB
+; RUN: not llubi --verbose --entry-function=oracle_poison_noundef_ret < %s 2>&1 | FileCheck %s --check-prefix=NOUNDEF-RET
+; RUN: not llubi --verbose --entry-function=oracle_poison_noundef_arg < %s 2>&1 | FileCheck %s --check-prefix=NOUNDEF-ARG
+
+ at a = global [6 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5]
+
+; Returns one element more than the number of bytes from %p to %end.
+define i64 @oracle_off_by_one(ptr %p, ptr %end) memory(none) nounwind nosync willreturn {
+ %p.int = ptrtoaddr ptr %p to i64
+ %end.int = ptrtoaddr ptr %end to i64
+ %diff = sub i64 %end.int, %p.int
+ %n = add i64 %diff, 4
+ ret i64 %n
+}
+
+define noundef i64 @oracle_noundef_ret(i64 %n) memory(none) nounwind nosync willreturn {
+ ret i64 poison
+}
+
+define i64 @oracle_noundef_arg(i64 noundef %n) memory(none) nounwind nosync willreturn {
+ ret i64 %n
+}
+
+define void @oracle_out_of_bounds() {
+; ORACLE-OOB: Entering function: oracle_out_of_bounds
+; ORACLE-OOB-NEXT: %p = getelementptr i32, ptr @a, i64 4 => ptr 0x20 [@a + 16]
+; ORACLE-OOB-NEXT: %end = getelementptr i32, ptr @a, i64 6 => ptr 0x28 [@a + 24]
+; ORACLE-OOB-NEXT: Entering function: oracle_off_by_one
+; ORACLE-OOB-NEXT: ptr %p = ptr 0x20 [@a + 16]
+; ORACLE-OOB-NEXT: ptr %end = ptr 0x28 [@a + 24]
+; ORACLE-OOB-NEXT: %p.int = ptrtoaddr ptr %p to i64 => i64 32
+; ORACLE-OOB-NEXT: %end.int = ptrtoaddr ptr %end to i64 => i64 40
+; ORACLE-OOB-NEXT: %diff = sub i64 %end.int, %p.int => i64 8
+; ORACLE-OOB-NEXT: %n = add i64 %diff, 4 => i64 12
+; ORACLE-OOB-NEXT: ret i64 %n
+; ORACLE-OOB-NEXT: Exiting function: oracle_off_by_one
+; ORACLE-OOB-NEXT: Stacktrace:
+; ORACLE-OOB-NEXT: #0 %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr %p, i1 false, ptr @oracle_off_by_one, ptr %p, ptr %end) at @oracle_out_of_bounds <stdin>:{{[0-9]+}}
+; ORACLE-OOB-NEXT: Immediate UB detected: Memory access is out of bounds. Accessed size: 12, Address: 0x20, Object base: 0x10, Object size: 24.
+; ORACLE-OOB-NEXT: error: Execution of function 'oracle_out_of_bounds' failed.
+ %p = getelementptr i32, ptr @a, i64 4
+ %end = getelementptr i32, ptr @a, i64 6
+ %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr %p, i1 false, ptr @oracle_off_by_one, ptr %p, ptr %end)
+ ret void
+}
+
+define void @oracle_poison_noundef_ret() {
+; NOUNDEF-RET: Entering function: oracle_poison_noundef_ret
+; NOUNDEF-RET-NEXT: Entering function: oracle_noundef_ret
+; NOUNDEF-RET-NEXT: i64 %n = i64 4
+; NOUNDEF-RET-NEXT: ret i64 poison
+; NOUNDEF-RET-NEXT: Exiting function: oracle_noundef_ret
+; NOUNDEF-RET-NEXT: Stacktrace:
+; NOUNDEF-RET-NEXT: #0 %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr @a, i1 false, ptr @oracle_noundef_ret, i64 4) at @oracle_poison_noundef_ret <stdin>:{{[0-9]+}}
+; NOUNDEF-RET-NEXT: Immediate UB detected: The value poison violates noundef attribute.
+; NOUNDEF-RET-NEXT: error: Execution of function 'oracle_poison_noundef_ret' failed.
+ %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr @a, i1 false, ptr @oracle_noundef_ret, i64 4)
+ ret void
+}
+
+define void @oracle_poison_noundef_arg() {
+; NOUNDEF-ARG: Entering function: oracle_poison_noundef_arg
+; NOUNDEF-ARG-NEXT: Stacktrace:
+; NOUNDEF-ARG-NEXT: #0 %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr @a, i1 false, ptr @oracle_noundef_arg, i64 poison) at @oracle_poison_noundef_arg <stdin>:{{[0-9]+}}
+; NOUNDEF-ARG-NEXT: Immediate UB detected: The value poison violates noundef attribute.
+; NOUNDEF-ARG-NEXT: error: Execution of function 'oracle_poison_noundef_arg' failed.
+ %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr @a, i1 false, ptr @oracle_noundef_arg, i64 poison)
+ ret void
+}
diff --git a/llvm/test/tools/llubi/intr_speculative_load_ub.ll b/llvm/test/tools/llubi/intr_speculative_load_ub.ll
index e08e1e410183d0..11ba81f804e1da 100644
--- a/llvm/test/tools/llubi/intr_speculative_load_ub.ll
+++ b/llvm/test/tools/llubi/intr_speculative_load_ub.ll
@@ -3,27 +3,14 @@
; RUN: not llubi --verbose --entry-function=exceeds_size < %s 2>&1 | FileCheck %s --check-prefix=EXCEEDS-SIZE
; RUN: not llubi --verbose --entry-function=poison_num_bytes < %s 2>&1 | FileCheck %s --check-prefix=POISON-N
; RUN: not llubi --verbose --entry-function=poison_pointer < %s 2>&1 | FileCheck %s --check-prefix=POISON-PTR
-; RUN: not llubi --verbose --entry-function=oracle_out_of_bounds < %s 2>&1 | FileCheck %s --check-prefix=ORACLE-OOB
-; RUN: not llubi --verbose --entry-function=oracle_declaration < %s 2>&1 | FileCheck %s --check-prefix=ORACLE-DECL
@a = global [2 x i32] [i32 0, i32 1]
-; Returns one element more than the number of bytes from %p to %end.
-define i64 @oracle_off_by_one(ptr %p, ptr %end) memory(none) nounwind nosync willreturn {
- %p.int = ptrtoaddr ptr %p to i64
- %end.int = ptrtoaddr ptr %end to i64
- %diff = sub i64 %end.int, %p.int
- %n = add i64 %diff, 4
- ret i64 %n
-}
-
-declare i64 @oracle_decl(i64) memory(none) nounwind nosync willreturn
-
define void @out_of_bounds() {
; OOB: Entering function: out_of_bounds
; OOB-NEXT: Stacktrace:
; OOB-NEXT: #0 %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr @a, i1 false, i64 12) at @out_of_bounds <stdin>:{{[0-9]+}}
-; OOB-NEXT: Immediate UB detected: Memory access is out of bounds. Accessed size: 12, Address: 0xc, Object base: 0xc, Object size: 8.
+; OOB-NEXT: Immediate UB detected: Memory access is out of bounds. Accessed size: 12, Address: 0x8, Object base: 0x8, Object size: 8.
; OOB-NEXT: error: Execution of function 'out_of_bounds' failed.
%r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr @a, i1 false, i64 12)
ret void
@@ -33,7 +20,7 @@ define void @from_end_out_of_bounds() {
; FROM-END-OOB: Entering function: from_end_out_of_bounds
; FROM-END-OOB-NEXT: Stacktrace:
; FROM-END-OOB-NEXT: #0 %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr getelementptr (i8, ptr @a, i64 -8), i1 true, i64 12) at @from_end_out_of_bounds <stdin>:{{[0-9]+}}
-; FROM-END-OOB-NEXT: Immediate UB detected: Memory access is out of bounds. Accessed size: 12, Address: 0x8, Object base: 0xc, Object size: 8.
+; FROM-END-OOB-NEXT: Immediate UB detected: Memory access is out of bounds. Accessed size: 12, Address: 0x4, Object base: 0x8, Object size: 8.
; FROM-END-OOB-NEXT: error: Execution of function 'from_end_out_of_bounds' failed.
%r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr getelementptr (i8, ptr @a, i64 -8), i1 true, i64 12)
ret void
@@ -68,19 +55,3 @@ define void @poison_pointer() {
%r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr poison, i1 false, i64 0)
ret void
}
-
-define void @oracle_out_of_bounds() {
-; ORACLE-OOB: Entering function: oracle_out_of_bounds
-; ORACLE-OOB-NEXT: Unrecognized instruction: %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr @a, i1 false, ptr @oracle_off_by_one, ptr @a, ptr getelementptr (i8, ptr @a, i64 8))
-; ORACLE-OOB-NEXT: error: Execution of function 'oracle_out_of_bounds' failed.
- %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr @a, i1 false, ptr @oracle_off_by_one, ptr @a, ptr getelementptr (i8, ptr @a, i64 8))
- ret void
-}
-
-define void @oracle_declaration() {
-; ORACLE-DECL: Entering function: oracle_declaration
-; ORACLE-DECL-NEXT: Unrecognized instruction: %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr @a, i1 false, ptr @oracle_decl, i64 4)
-; ORACLE-DECL-NEXT: error: Execution of function 'oracle_declaration' failed.
- %r = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr @a, i1 false, ptr @oracle_decl, i64 4)
- ret void
-}
diff --git a/llvm/tools/llubi/lib/Interpreter.cpp b/llvm/tools/llubi/lib/Interpreter.cpp
index f3071cf1d7464b..76f2535abbce42 100644
--- a/llvm/tools/llubi/lib/Interpreter.cpp
+++ b/llvm/tools/llubi/lib/Interpreter.cpp
@@ -919,6 +919,14 @@ class InstExecutor : public InstVisitor<InstExecutor, void>,
return AnyValue();
}
+ /// Returns the oracle function if \p CB is an llvm.speculative.load in
+ /// oracle form, nullptr otherwise.
+ static Function *getSpeculativeLoadOracle(const CallBase &CB) {
+ return CB.getIntrinsicID() == Intrinsic::speculative_load
+ ? dyn_cast<Function>(CB.getArgOperand(2))
+ : nullptr;
+ }
+
AnyValue callSpeculativeLoadIntrinsic(CallBase &CB, const AnyValue &Ptr,
const AnyValue &NumBytes) {
Type *RetTy = CB.getType();
@@ -1043,8 +1051,17 @@ class InstExecutor : public InstVisitor<InstExecutor, void>,
void returnFromCallee() {
auto &CB = cast<CallBase>(*CurrentFrame->PC);
- CurrentFrame->CalleeArgs.clear();
AnyValue &RetVal = CurrentFrame->CalleeRetVal;
+ if (Function *Oracle = getSpeculativeLoadOracle(CB)) {
+ // RetVal is the oracle's result; use it to complete the load.
+ handleAttributes(Oracle->getReturnType(), RetVal, AttributeSet(),
+ Oracle->getAttributes().getRetAttrs());
+ RetVal =
+ callSpeculativeLoadIntrinsic(CB, CurrentFrame->CalleeArgs[0], RetVal);
+ if (hasProgramExited())
+ return;
+ }
+ CurrentFrame->CalleeArgs.clear();
if (Type *RetTy = CB.getType(); !RetTy->isVoidTy()) {
// Handle attributes on the return value (Attributes from resolved callee
// should be applied if available).
@@ -1800,12 +1817,8 @@ class InstExecutor : public InstVisitor<InstExecutor, void>,
case Intrinsic::memset_inline:
return callMemSetIntrinsic(CB, Args);
case Intrinsic::speculative_load:
- // TODO: Support the oracle form.
- if (isa<Function>(CB.getArgOperand(2))) {
- Handler.onUnrecognizedInstruction(CB);
- setFailed();
- return AnyValue();
- }
+ assert(!getSpeculativeLoadOracle(CB) &&
+ "oracle form must be handled earlier");
return callSpeculativeLoadIntrinsic(CB, Args[0], Args[2]);
case Intrinsic::experimental_noalias_scope_decl:
// FIXME: Not implemented yet. Currently it acts as a noop.
@@ -2228,6 +2241,15 @@ class InstExecutor : public InstVisitor<InstExecutor, void>,
}
CurrentFrame->ResolvedCallee = Callee;
+ ArrayRef<AnyValue> Args = CalleeArgs;
+ if (Function *Oracle = getSpeculativeLoadOracle(CB)) {
+ Args = Args.drop_front(3);
+ for (auto [Arg, ArgVal] :
+ zip_equal(Oracle->args(), MutableArrayRef(CalleeArgs).drop_front(3)))
+ handleAttributes(Arg.getType(), ArgVal, AttributeSet(),
+ Arg.getAttributes());
+ Callee = Oracle;
+ }
if (Callee->isIntrinsic()) {
CurrentFrame->CalleeRetVal = callIntrinsic(CB, CalleeArgs);
returnFromCallee();
@@ -2244,7 +2266,6 @@ class InstExecutor : public InstVisitor<InstExecutor, void>,
}
assert(!Callee->empty() && "Expected a defined function.");
// Suspend the current frame and push the callee frame onto the stack.
- ArrayRef<AnyValue> Args = CurrentFrame->CalleeArgs;
AnyValue &RetVal = CurrentFrame->CalleeRetVal;
CurrentFrame->State = FrameState::Pending;
CallStack.emplace_back(*Callee, &CB, CurrentFrame, Args, RetVal,
>From dc8bd456d79aca2e79f1cea2130ec02cf44b8f73 Mon Sep 17 00:00:00 2001
From: Julian Nagele <j.nagele at apple.com>
Date: Tue, 6 Oct 2026 22:30:56 +0100
Subject: [PATCH 27/46] [LV] Add tests for uniform inner loops in outer-loop
path (NFC) (#227415)
Add tests for outer-loop vectorization, where the inner loop is uniform:
every lane computes the same inner latch condition.
Since #199632, the legality check accepts an inner-loop latch whose
condition is an icmp with both operands SCEV loop-uniform in the outer
loop. These tests document that new behaviour.
---
.../outer_loop_uniform_inner_loop.ll | 694 ++++++++++++++++++
1 file changed, 694 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/outer_loop_uniform_inner_loop.ll
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_uniform_inner_loop.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_uniform_inner_loop.ll
new file mode 100644
index 00000000000000..9ffbd9eea661cd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_uniform_inner_loop.ll
@@ -0,0 +1,694 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
+; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s
+
+; Outer-loop vectorization of nests whose inner loop is uniform: every lane
+; computes the same inner latch condition. The inner IV need not be canonical,
+; and the inner trip count need not be computable.
+
+; for (i = 0; i < N; i++)
+; for (j = start; j < M; j++)
+; A[i*M+j] = 0;
+define void @inner_start_invariant(ptr %A, i64 %N, i64 %M, i64 %start) {
+; CHECK-LABEL: define void @inner_start_invariant(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]], i64 [[START:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[START]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH5:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH5]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[INNER_BODY3:.*]]
+; CHECK: [[INNER_BODY3]]:
+; CHECK-NEXT: [[INNER_IV4:%.*]] = phi <4 x i64> [ [[BROADCAST_SPLAT2]], %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER_BODY3]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i64> [[TMP1]], [[INNER_IV4]]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP3]] = add nsw <4 x i64> [[INNER_IV4]], splat (i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[OUTER_LATCH5]], label %[[INNER_BODY3]]
+; CHECK: [[OUTER_LATCH5]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %outer.offset = mul nsw i64 %outer.iv, %M
+ br label %inner.body
+
+inner.body:
+ %inner.iv = phi i64 [ %start, %outer.header ], [ %inner.iv.next, %inner.body ]
+ %idx = add nsw i64 %outer.offset, %inner.iv
+ %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
+ store float 0.000000e+00, ptr %A.ptr, align 4
+ %inner.iv.next = add nsw i64 %inner.iv, 1
+ %inner.cmp = icmp eq i64 %inner.iv.next, %M
+ br i1 %inner.cmp, label %outer.latch, label %inner.body
+
+outer.latch:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.cmp = icmp eq i64 %outer.iv.next, %N
+ br i1 %outer.cmp, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+; for (i = 0; i < N; i++)
+; for (j = 0; j < M; j += 2)
+; A[i*M+j] = 0;
+define void @inner_step_2(ptr %A, i64 %N, i64 %M) {
+; CHECK-LABEL: define void @inner_step_2(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH3:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH3]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[INNER_BODY1:.*]]
+; CHECK: [[INNER_BODY1]]:
+; CHECK-NEXT: [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER_BODY1]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i64> [[TMP1]], [[INNER_IV2]]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP3]] = add nsw <4 x i64> [[INNER_IV2]], splat (i64 2)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp sge <4 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[OUTER_LATCH3]], label %[[INNER_BODY1]]
+; CHECK: [[OUTER_LATCH3]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %outer.offset = mul nsw i64 %outer.iv, %M
+ br label %inner.body
+
+inner.body:
+ %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
+ %idx = add nsw i64 %outer.offset, %inner.iv
+ %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
+ store float 0.000000e+00, ptr %A.ptr, align 4
+ %inner.iv.next = add nsw i64 %inner.iv, 2
+ %inner.cmp = icmp sge i64 %inner.iv.next, %M
+ br i1 %inner.cmp, label %outer.latch, label %inner.body
+
+outer.latch:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.cmp = icmp eq i64 %outer.iv.next, %N
+ br i1 %outer.cmp, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+; The latch compares a truncated IV update.
+; for (i = 0; i < N; i++)
+; for (long j = start; (int)j != bound; j++)
+; A[i*64+j] = 0;
+define void @inner_latch_cmp_on_trunc(ptr %A, i64 %N, i32 %bound, i64 %start) {
+; CHECK-LABEL: define void @inner_latch_cmp_on_trunc(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[BOUND:%.*]], i64 [[START:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[BOUND]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[START]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH5:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH5]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = shl nsw <4 x i64> [[VEC_IND]], splat (i64 6)
+; CHECK-NEXT: br label %[[INNER_BODY3:.*]]
+; CHECK: [[INNER_BODY3]]:
+; CHECK-NEXT: [[INNER_IV4:%.*]] = phi <4 x i64> [ [[BROADCAST_SPLAT2]], %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER_BODY3]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i64> [[TMP1]], [[INNER_IV4]]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP3]] = add nsw <4 x i64> [[INNER_IV4]], splat (i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i64> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i32> [[BROADCAST_SPLAT]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: br i1 [[TMP6]], label %[[OUTER_LATCH5]], label %[[INNER_BODY3]]
+; CHECK: [[OUTER_LATCH5]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %outer.offset = mul nsw i64 %outer.iv, 64
+ br label %inner.body
+
+inner.body:
+ %inner.iv = phi i64 [ %start, %outer.header ], [ %inner.iv.next, %inner.body ]
+ %idx = add nsw i64 %outer.offset, %inner.iv
+ %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
+ store float 0.000000e+00, ptr %A.ptr, align 4
+ %inner.iv.next = add nsw i64 %inner.iv, 1
+ %inner.iv.next.trunc = trunc i64 %inner.iv.next to i32
+ %inner.cmp = icmp eq i32 %bound, %inner.iv.next.trunc
+ br i1 %inner.cmp, label %outer.latch, label %inner.body
+
+outer.latch:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.cmp = icmp eq i64 %outer.iv.next, %N
+ br i1 %outer.cmp, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+; The inner bound is computed inside the outer loop, but its SCEV is invariant
+; in the outer loop.
+; for (i = 0; i < N; i++) {
+; long bound = M + 1;
+; for (j = 0; j < bound; j++)
+; A[i*M+j] = 0;
+; }
+define void @inner_bound_defined_in_outer_loop(ptr %A, i64 %N, i64 %M) {
+; CHECK-LABEL: define void @inner_bound_defined_in_outer_loop(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw <4 x i64> [[BROADCAST_SPLAT]], splat (i64 1)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH3:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH3]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[INNER_BODY1:.*]]
+; CHECK: [[INNER_BODY1]]:
+; CHECK-NEXT: [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_BODY1]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i64> [[TMP2]], [[INNER_IV2]]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP3]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: br i1 [[TMP6]], label %[[OUTER_LATCH3]], label %[[INNER_BODY1]]
+; CHECK: [[OUTER_LATCH3]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %outer.offset = mul nsw i64 %outer.iv, %M
+ %bound = add nsw i64 %M, 1
+ br label %inner.body
+
+inner.body:
+ %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
+ %idx = add nsw i64 %outer.offset, %inner.iv
+ %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
+ store float 0.000000e+00, ptr %A.ptr, align 4
+ %inner.iv.next = add nuw nsw i64 %inner.iv, 1
+ %inner.cmp = icmp eq i64 %inner.iv.next, %bound
+ br i1 %inner.cmp, label %outer.latch, label %inner.body
+
+outer.latch:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.cmp = icmp eq i64 %outer.iv.next, %N
+ br i1 %outer.cmp, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+; Triangular 3-level nest. The innermost bound is an IV of the middle loop, so
+; it is the same in every lane.
+; for (i = 0; i < N; i++)
+; for (j = 0; j < M; j++)
+; for (k = 0; k <= j; k++)
+; A[i*M+k] = 0;
+define void @nested_inner_bound_is_middle_iv(ptr %A, i64 %N, i64 %M) {
+; CHECK-LABEL: define void @nested_inner_bound_is_middle_iv(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH6:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH6]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[MIDDLE_HEADER1:.*]]
+; CHECK: [[MIDDLE_HEADER1]]:
+; CHECK-NEXT: [[MIDDLE_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP6:%.*]], %[[MIDDLE_LATCH5:.*]] ]
+; CHECK-NEXT: br label %[[INNER_BODY3:.*]]
+; CHECK: [[INNER_BODY3]]:
+; CHECK-NEXT: [[INNER_IV4:%.*]] = phi <4 x i64> [ zeroinitializer, %[[MIDDLE_HEADER1]] ], [ [[TMP3:%.*]], %[[INNER_BODY3]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i64> [[TMP1]], [[INNER_IV4]]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP3]] = add nuw nsw <4 x i64> [[INNER_IV4]], splat (i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <4 x i64> [[TMP3]], [[MIDDLE_IV2]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_LATCH5]], label %[[INNER_BODY3]]
+; CHECK: [[MIDDLE_LATCH5]]:
+; CHECK-NEXT: [[TMP6]] = add nuw nsw <4 x i64> [[MIDDLE_IV2]], splat (i64 1)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
+; CHECK-NEXT: br i1 [[TMP8]], label %[[OUTER_LATCH6]], label %[[MIDDLE_HEADER1]]
+; CHECK: [[OUTER_LATCH6]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %outer.offset = mul nsw i64 %outer.iv, %M
+ br label %middle.header
+
+middle.header:
+ %middle.iv = phi i64 [ 0, %outer.header ], [ %middle.iv.next, %middle.latch ]
+ br label %inner.body
+
+inner.body:
+ %inner.iv = phi i64 [ 0, %middle.header ], [ %inner.iv.next, %inner.body ]
+ %idx = add nsw i64 %outer.offset, %inner.iv
+ %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
+ store float 0.000000e+00, ptr %A.ptr, align 4
+ %inner.iv.next = add nuw nsw i64 %inner.iv, 1
+ %inner.cmp = icmp ugt i64 %inner.iv.next, %middle.iv
+ br i1 %inner.cmp, label %middle.latch, label %inner.body
+
+middle.latch:
+ %middle.iv.next = add nuw nsw i64 %middle.iv, 1
+ %middle.cmp = icmp eq i64 %middle.iv.next, %M
+ br i1 %middle.cmp, label %outer.latch, label %middle.header
+
+outer.latch:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.cmp = icmp eq i64 %outer.iv.next, %N
+ br i1 %outer.cmp, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+; The inner latch continues while j.next == X. SCEV does not model this exit.
+define void @inner_latch_continue_while_eq(ptr %A, i64 %N, i64 %M, i32 %X) {
+; CHECK-LABEL: define void @inner_latch_continue_while_eq(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]], i32 [[X:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH5:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH5]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[INNER_BODY3:.*]]
+; CHECK: [[INNER_BODY3]]:
+; CHECK-NEXT: [[INNER_IV4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_BODY3]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i32> [[INNER_IV4]] to <4 x i64>
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP3]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP4]] = add <4 x i32> [[INNER_IV4]], splat (i32 1)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <4 x i32> [[TMP4]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: br i1 [[TMP6]], label %[[OUTER_LATCH5]], label %[[INNER_BODY3]]
+; CHECK: [[OUTER_LATCH5]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %outer.offset = mul nsw i64 %outer.iv, %M
+ br label %inner.body
+
+inner.body:
+ %inner.iv = phi i32 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
+ %inner.iv.ext = zext i32 %inner.iv to i64
+ %idx = add nsw i64 %outer.offset, %inner.iv.ext
+ %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
+ store float 0.000000e+00, ptr %A.ptr, align 4
+ %inner.iv.next = add i32 %inner.iv, 1
+ %inner.cont = icmp eq i32 %inner.iv.next, %X
+ br i1 %inner.cont, label %inner.body, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.cmp = icmp eq i64 %outer.iv.next, %N
+ br i1 %outer.cmp, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+; The inner latch continues while j.next <=u X. The loop does not terminate
+; if X is UINT_MAX, so SCEV cannot compute its trip count.
+define void @inner_latch_continue_while_ule(ptr %A, i64 %N, i64 %M, i32 %X) {
+; CHECK-LABEL: define void @inner_latch_continue_while_ule(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]], i32 [[X:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH5:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH5]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[INNER_BODY3:.*]]
+; CHECK: [[INNER_BODY3]]:
+; CHECK-NEXT: [[INNER_IV4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_BODY3]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i32> [[INNER_IV4]] to <4 x i64>
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP3]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP4]] = add <4 x i32> [[INNER_IV4]], splat (i32 1)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ugt <4 x i32> [[TMP4]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: br i1 [[TMP6]], label %[[OUTER_LATCH5]], label %[[INNER_BODY3]]
+; CHECK: [[OUTER_LATCH5]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %outer.offset = mul nsw i64 %outer.iv, %M
+ br label %inner.body
+
+inner.body:
+ %inner.iv = phi i32 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
+ %inner.iv.ext = zext i32 %inner.iv to i64
+ %idx = add nsw i64 %outer.offset, %inner.iv.ext
+ %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
+ store float 0.000000e+00, ptr %A.ptr, align 4
+ %inner.iv.next = add i32 %inner.iv, 1
+ %inner.cont = icmp ule i32 %inner.iv.next, %X
+ br i1 %inner.cont, label %inner.body, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.cmp = icmp eq i64 %outer.iv.next, %N
+ br i1 %outer.cmp, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+; The inner loop is controlled by a pointer IV. Its start and end are the same
+; in every lane.
+; for (i = 0; i < N; i++) {
+; float s = 0;
+; for (const float *p = begin; p != end; ++p)
+; s += *p;
+; A[i] = s;
+; }
+define void @inner_latch_ptr_iv(ptr noalias %A, ptr %begin, ptr %end, i64 %N) {
+; CHECK-LABEL: define void @inner_latch_ptr_iv(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[BEGIN:%.*]], ptr [[END:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x ptr> poison, ptr [[END]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x ptr> poison, ptr [[BEGIN]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT1]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH6:.*]] ]
+; CHECK-NEXT: br label %[[INNER_BODY3:.*]]
+; CHECK: [[INNER_BODY3]]:
+; CHECK-NEXT: [[INNER_IV4:%.*]] = phi <4 x ptr> [ [[BROADCAST_SPLAT2]], %[[VECTOR_BODY]] ], [ [[WIDE_GEP:%.*]], %[[INNER_BODY3]] ]
+; CHECK-NEXT: [[S5:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP1:%.*]], %[[INNER_BODY3]] ]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[INNER_IV4]], <4 x i1> splat (i1 true), <4 x float> poison)
+; CHECK-NEXT: [[TMP1]] = fadd <4 x float> [[S5]], [[WIDE_MASKED_GATHER]]
+; CHECK-NEXT: [[WIDE_GEP]] = getelementptr inbounds float, <4 x ptr> [[INNER_IV4]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x ptr> [[WIDE_GEP]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[OUTER_LATCH6]], label %[[INNER_BODY3]]
+; CHECK: [[OUTER_LATCH6]]:
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x float> [[TMP1]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ br label %inner.body
+
+inner.body:
+ %inner.iv = phi ptr [ %begin, %outer.header ], [ %inner.iv.next, %inner.body ]
+ %s = phi float [ 0.000000e+00, %outer.header ], [ %s.next, %inner.body ]
+ %v = load float, ptr %inner.iv, align 4
+ %s.next = fadd float %s, %v
+ %inner.iv.next = getelementptr inbounds float, ptr %inner.iv, i64 1
+ %inner.cmp = icmp eq ptr %inner.iv.next, %end
+ br i1 %inner.cmp, label %outer.latch, label %inner.body
+
+outer.latch:
+ %s.lcssa = phi float [ %s.next, %inner.body ]
+ %A.ptr = getelementptr inbounds float, ptr %A, i64 %outer.iv
+ store float %s.lcssa, ptr %A.ptr, align 4
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.cmp = icmp eq i64 %outer.iv.next, %N
+ br i1 %outer.cmp, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+; The inner latch is an fcmp. Only icmp latch conditions are checked for
+; uniformity with SCEV, so the loop is rejected even though it is uniform.
+; for (i = 0; i < N; i++)
+; for (float x = 0; x < X; x += 1)
+; A[i] = x;
+define void @inner_latch_fp_cmp(ptr %A, i64 %N, float %X) {
+; CHECK-LABEL: define void @inner_latch_fp_cmp(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]], float [[X:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[OUTER_IV]]
+; CHECK-NEXT: br label %[[INNER_BODY:.*]]
+; CHECK: [[INNER_BODY]]:
+; CHECK-NEXT: [[INNER_IV:%.*]] = phi float [ 0.000000e+00, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
+; CHECK-NEXT: store float [[INNER_IV]], ptr [[A_PTR]], align 4
+; CHECK-NEXT: [[INNER_IV_NEXT]] = fadd float [[INNER_IV]], 1.000000e+00
+; CHECK-NEXT: [[INNER_CONT:%.*]] = fcmp olt float [[INNER_IV_NEXT]], [[X]]
+; CHECK-NEXT: br i1 [[INNER_CONT]], label %[[INNER_BODY]], label %[[OUTER_LATCH]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
+; CHECK-NEXT: [[OUTER_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[OUTER_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %A.ptr = getelementptr inbounds float, ptr %A, i64 %outer.iv
+ br label %inner.body
+
+inner.body:
+ %inner.iv = phi float [ 0.000000e+00, %outer.header ], [ %inner.iv.next, %inner.body ]
+ store float %inner.iv, ptr %A.ptr, align 4
+ %inner.iv.next = fadd float %inner.iv, 1.000000e+00
+ %inner.cont = fcmp olt float %inner.iv.next, %X
+ br i1 %inner.cont, label %inner.body, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.cmp = icmp eq i64 %outer.iv.next, %N
+ br i1 %outer.cmp, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+; TODO: The inner loop is uniform, because every lane runs M iterations, but it
+; is not recognized: the inner IV start and bound both depend on the outer IV,
+; so the compared values differ per lane.
+; for (i = 0; i < N; i++) {
+; float s = 0;
+; for (j = i; j < i + M; j++)
+; s += B[j];
+; A[i] = s;
+; }
+define void @inner_start_and_bound_shift_with_outer(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M) {
+; CHECK-LABEL: define void @inner_start_and_bound_shift_with_outer(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: [[END:%.*]] = add nsw i64 [[OUTER_IV]], [[M]]
+; CHECK-NEXT: br label %[[INNER_BODY:.*]]
+; CHECK: [[INNER_BODY]]:
+; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ [[OUTER_IV]], %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
+; CHECK-NEXT: [[S:%.*]] = phi float [ 0.000000e+00, %[[OUTER_HEADER]] ], [ [[S_NEXT:%.*]], %[[INNER_BODY]] ]
+; CHECK-NEXT: [[B_PTR:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INNER_IV]]
+; CHECK-NEXT: [[B_VAL:%.*]] = load float, ptr [[B_PTR]], align 4
+; CHECK-NEXT: [[S_NEXT]] = fadd float [[S]], [[B_VAL]]
+; CHECK-NEXT: [[INNER_IV_NEXT]] = add nsw i64 [[INNER_IV]], 1
+; CHECK-NEXT: [[INNER_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[END]]
+; CHECK-NEXT: br i1 [[INNER_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[S_LCSSA:%.*]] = phi float [ [[S_NEXT]], %[[INNER_BODY]] ]
+; CHECK-NEXT: [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[OUTER_IV]]
+; CHECK-NEXT: store float [[S_LCSSA]], ptr [[A_PTR]], align 4
+; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
+; CHECK-NEXT: [[OUTER_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[OUTER_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP18]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %end = add nsw i64 %outer.iv, %M
+ br label %inner.body
+
+inner.body:
+ %inner.iv = phi i64 [ %outer.iv, %outer.header ], [ %inner.iv.next, %inner.body ]
+ %s = phi float [ 0.000000e+00, %outer.header ], [ %s.next, %inner.body ]
+ %B.ptr = getelementptr inbounds float, ptr %B, i64 %inner.iv
+ %B.val = load float, ptr %B.ptr, align 4
+ %s.next = fadd float %s, %B.val
+ %inner.iv.next = add nsw i64 %inner.iv, 1
+ %inner.cmp = icmp eq i64 %inner.iv.next, %end
+ br i1 %inner.cmp, label %outer.latch, label %inner.body
+
+outer.latch:
+ %s.lcssa = phi float [ %s.next, %inner.body ]
+ %A.ptr = getelementptr inbounds float, ptr %A, i64 %outer.iv
+ store float %s.lcssa, ptr %A.ptr, align 4
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.cmp = icmp eq i64 %outer.iv.next, %N
+ br i1 %outer.cmp, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.vectorize.width", i32 4}
+!2 = !{!"llvm.loop.vectorize.enable"}
>From 83b71417a0fbf29f8eed8f0aafad695cef687dbb Mon Sep 17 00:00:00 2001
From: Sam Elliott <aelliott at qti.qualcomm.com>
Date: Tue, 6 Oct 2026 14:34:13 -0700
Subject: [PATCH 28/46] [clang][Driver] Support Trailing Comments in Config
Files (#213765)
Before this change, if users put trailing comments into a config file,
these would be added verbatim as processed arguments.
This change ensures that trailing comments after flags are not added to
the processed arguments. This means that comments in config files work
more closely to how comments in unix shell scripts work.
This change is only applied to config files (i.e. from `--config=file`),
and not GNU response files (i.e. from `@file`).
Co-authored-by: Fangrui Song <i at maskray.me>
---
clang/docs/ReleaseNotes.md | 5 ++
clang/docs/UsersManual.md | 15 ++--
llvm/lib/Support/CommandLine.cpp | 32 +++++++-
llvm/unittests/Support/CommandLineTest.cpp | 86 ++++++++++++++++++++++
4 files changed, 129 insertions(+), 9 deletions(-)
diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md
index 25299915441501..08442f74879afc 100644
--- a/clang/docs/ReleaseNotes.md
+++ b/clang/docs/ReleaseNotes.md
@@ -305,6 +305,11 @@ features cannot lower the translation-unit ABI level;
`-ftrivial-auto-var-init=` entry in the User's Manual for the details,
including where C deliberately departs from C 6.2.4p6.
+- `--config` files now support trailing `#` comments after an option, in
+ addition to whole-line comments. Mid-line comment-denoting `#`s must be
+ separated from surrounding arguments by whitespace. Quoted arguments can no
+ longer be split across multiple lines using a backslash.
+
### Removed Compiler Flags
### Attribute Changes in Clang
diff --git a/clang/docs/UsersManual.md b/clang/docs/UsersManual.md
index 7a0bd54c219fbb..96d1c4e009c71a 100644
--- a/clang/docs/UsersManual.md
+++ b/clang/docs/UsersManual.md
@@ -1138,11 +1138,14 @@ x86_64-pc-linux-gnu.cfg
It is not an error if either of these files is not found.
-The configuration file consists of command-line options specified on one or
-more lines. Lines composed of whitespace characters only are ignored as well as
-lines in which the first non-blank character is `#`. Long options may be split
-between several lines by a trailing backslash. Here is an example of a
-configuration file:
+The configuration file consists of command-line options specified on one or more
+lines. Lines composed of whitespace characters only are ignored as well as lines
+in which the first non-blank character is `#`. A `#` elsewhere on a line also
+starts a comment that runs to the end of the line, as long as it begins a new
+word (i.e. is preceded by whitespace) and is not inside a quoted string. Long
+options may be split between several lines by a trailing backslash.
+
+Here is an example of a configuration file:
```
# Several options on line
@@ -1154,6 +1157,8 @@ include/c++/5.4.0
# other config files may be included
@linux.options
+
+-Wall # trailing comments are supported
```
Files included by `@file` directives in configuration files are resolved
diff --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp
index 336ac2018e5b9a..c2ac06dd6232b0 100644
--- a/llvm/lib/Support/CommandLine.cpp
+++ b/llvm/lib/Support/CommandLine.cpp
@@ -1043,10 +1043,17 @@ void cl::tokenizeConfigFile(StringRef Source, StringSaver &Saver,
++Cur;
continue;
}
- // Find end of the current line.
+ // Find end of the current line, splicing backslash-newline continuations.
+ // A '#' that begins a new unquoted token starts a comment running to the
+ // next literal newline; escapes and continuations are not honored inside
+ // the comment.
const char *Start = Cur;
+ const char *LineEnd = nullptr;
+ char Quote = 0;
+ bool AtTokenStart = true;
for (const char *End = Source.end(); Cur != End; ++Cur) {
- if (*Cur == '\\') {
+ char C = *Cur;
+ if (C == '\\') {
if (Cur + 1 != End) {
++Cur;
if (*Cur == '\n' ||
@@ -1055,13 +1062,30 @@ void cl::tokenizeConfigFile(StringRef Source, StringSaver &Saver,
if (*Cur == '\r')
++Cur;
Start = Cur + 1;
+ // Splicing does not introduce a token boundary.
+ continue;
}
}
- } else if (*Cur == '\n')
+ AtTokenStart = false;
+ continue;
+ }
+ if (C == '\n')
break;
+ if (Quote) {
+ if (C == Quote)
+ Quote = 0;
+ } else if (isQuote(C)) {
+ Quote = C;
+ } else if (C == '#' && AtTokenStart) {
+ LineEnd = Cur;
+ while (Cur != End && *Cur != '\n')
+ ++Cur;
+ break;
+ }
+ AtTokenStart = isWhitespace(C);
}
// Tokenize line.
- Line.append(Start, Cur);
+ Line.append(Start, LineEnd ? LineEnd : Cur);
cl::TokenizeGNUCommandLine(Line, Saver, NewArgv, MarkEOLs);
}
}
diff --git a/llvm/unittests/Support/CommandLineTest.cpp b/llvm/unittests/Support/CommandLineTest.cpp
index 639f4f2ea8dc85..fc5d93acf87b86 100644
--- a/llvm/unittests/Support/CommandLineTest.cpp
+++ b/llvm/unittests/Support/CommandLineTest.cpp
@@ -438,6 +438,92 @@ TEST(CommandLineTest, TokenizeConfigFile11) {
testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
}
+TEST(CommandLineTest, TokenizeConfigFileTrailingComment) {
+ const char *Input = "-c # comment\n";
+ const char *const Output[] = {"-c"};
+ testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
+}
+
+TEST(CommandLineTest, TokenizeConfigFileTrailingCommentNoNewline) {
+ const char *Input = "-c # comment";
+ const char *const Output[] = {"-c"};
+ testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
+}
+
+TEST(CommandLineTest, TokenizeConfigFileLineAfterComment) {
+ const char *Input = "-a # comment\n-b\n";
+ const char *const Output[] = {"-a", "-b"};
+ testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
+}
+
+TEST(CommandLineTest, TokenizeConfigFileNoContinuationInComment) {
+ // A backslash-newline inside a comment is not a continuation.
+ const char *Input = "-c # comment \\\n-d\n";
+ const char *const Output[] = {"-c", "-d"};
+ testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
+}
+
+TEST(CommandLineTest, TokenizeConfigFileHashNotAtTokenStart) {
+ // '#' not preceded by whitespace is not a comment.
+ const char *Input = "-DFOO=1#2\n";
+ const char *const Output[] = {"-DFOO=1#2"};
+ testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
+}
+
+TEST(CommandLineTest, TokenizeConfigFileHashInQuotes) {
+ const char *Input = "-DFOO=\"a # b\" # comment\n";
+ const char *const Output[] = {"-DFOO=a # b"};
+ testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
+}
+
+TEST(CommandLineTest, TokenizeConfigFileHashAfterClosingQuote) {
+ const char *Input = "-DFOO=\"a\"#b\n";
+ const char *const Output[] = {"-DFOO=a#b"};
+ testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
+}
+
+TEST(CommandLineTest, TokenizeConfigFileNewlineEndsQuote) {
+ // A literal newline ends the line even inside a quoted string.
+ const char *Input = "-DA=\"x\n-DB=y\n";
+ const char *const Output[] = {"-DA=x", "-DB=y"};
+ testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
+}
+
+TEST(CommandLineTest, TokenizeConfigFileEscapedHash) {
+ const char *Input = "-c \\#comment\n";
+ const char *const Output[] = {"-c", "#comment"};
+ testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
+}
+
+TEST(CommandLineTest, TokenizeConfigFileCommentAfterContinuationWithSpace) {
+ // The splice keeps the preceding whitespace in effect, so the '#' begins a
+ // token and starts a comment.
+ const char *Input = "-c \\\n# comment\n";
+ const char *const Output[] = {"-c"};
+ testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
+}
+
+TEST(CommandLineTest, TokenizeConfigFileHashAfterContinuationNoSpace) {
+ // No whitespace before the splice: '#' continues the token (cf.
+ // TokenizeConfigFile4).
+ const char *Input = "abc\\\n#comment\n";
+ const char *const Output[] = {"abc#comment"};
+ testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
+}
+
+TEST(CommandLineTest, TokenizeConfigFileCommentAfterContinuationWithOption) {
+ const char *Input = "-c \\\noption # comment\n";
+ const char *const Output[] = {"-c", "option"};
+ testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
+}
+
+TEST(CommandLineTest, TokenizeConfigFileHashesInFlagNotComment) {
+ // None of the '#'s in -### begins a token.
+ const char *Input = "-### # comment\n";
+ const char *const Output[] = {"-###"};
+ testCommandLineTokenizer(cl::tokenizeConfigFile, Input, Output);
+}
+
TEST(CommandLineTest, AliasesWithArguments) {
static const size_t ARGC = 3;
const char *const Inputs[][ARGC] = {
>From c875fb1aac2c7adfd9e7456a63de63d35bd11e0c Mon Sep 17 00:00:00 2001
From: Gregory Roth <groth at nvidia.com>
Date: Tue, 6 Oct 2026 14:35:59 -0700
Subject: [PATCH 29/46] [HLSL] Compatibility overloads for scalar min/max
(#222746)
A lot of shaders fail with ambiguous overloads using int and float
together in min or max calls. This just adds the usual compatibility
overload with the usual warning for those including testing
---
clang/lib/Headers/hlsl/hlsl_compat_overloads.h | 16 ++++++++++++++++
.../test/CodeGenHLSL/builtins/max-overloads.hlsl | 12 ++++++++++++
.../test/CodeGenHLSL/builtins/min-overloads.hlsl | 12 ++++++++++++
3 files changed, 40 insertions(+)
diff --git a/clang/lib/Headers/hlsl/hlsl_compat_overloads.h b/clang/lib/Headers/hlsl/hlsl_compat_overloads.h
index 2c0c7677be36fc..53ea7e28ef41da 100644
--- a/clang/lib/Headers/hlsl/hlsl_compat_overloads.h
+++ b/clang/lib/Headers/hlsl/hlsl_compat_overloads.h
@@ -28,6 +28,10 @@ namespace hlsl {
[[deprecated("In 202x mismatched vector/scalar lowering for " #fn " is " \
"deprecated. Explicitly cast parameters.")]]
+#define _DXC_DEPRECATED_SCALAR_FN(ty1, ty2, fn) \
+ [[deprecated("In 202x mismatched " #ty1 "/" #ty2 " lowering for " #fn " is " \
+ "deprecated. Explicitly cast parameters.")]]
+
#define _DXC_COMPAT_UNARY_DOUBLE_OVERLOADS(fn) \
_DXC_DEPRECATED_64BIT_FN(fn) \
constexpr float fn(double V) { return fn((float)V); } \
@@ -517,6 +521,12 @@ constexpr __detail::enable_if_t<(N > 1 && N <= 4), vector<T, N>> max(
return max((vector<T, N>)p0, p1);
}
+_DXC_DEPRECATED_SCALAR_FN(float, int, max)
+constexpr float max(float p0, int p1) { return max(p0, (float)p1); }
+
+_DXC_DEPRECATED_SCALAR_FN(int, float, max)
+constexpr float max(int p0, float p1) { return max((float)p0, p1); }
+
//===----------------------------------------------------------------------===//
// min builtins overloads
//===----------------------------------------------------------------------===//
@@ -535,6 +545,12 @@ constexpr __detail::enable_if_t<(N > 1 && N <= 4), vector<T, N>> min(
return min((vector<T, N>)p0, p1);
}
+_DXC_DEPRECATED_SCALAR_FN(float, int, min)
+constexpr float min(float p0, int p1) { return min(p0, (float)p1); }
+
+_DXC_DEPRECATED_SCALAR_FN(int, float, min)
+constexpr float min(int p0, float p1) { return min((float)p0, p1); }
+
//===----------------------------------------------------------------------===//
// normalize builtins overloads
//===----------------------------------------------------------------------===//
diff --git a/clang/test/CodeGenHLSL/builtins/max-overloads.hlsl b/clang/test/CodeGenHLSL/builtins/max-overloads.hlsl
index da5cd8ff375100..0f437eecd0af0f 100644
--- a/clang/test/CodeGenHLSL/builtins/max-overloads.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/max-overloads.hlsl
@@ -79,3 +79,15 @@ double4 test_max_double4_mismatch(double4 p0, double p1) { return max(p0, p1); }
// CHECK: [[MAX:%.*]] = call reassoc nnan ninf nsz arcp afn noundef nofpclass(nan inf) <4 x double> @llvm.maxnum.v4f64(<4 x double> [[CONV1]], <4 x double> %{{.*}})
// CHECK: ret <4 x double> [[MAX]]
double4 test_max_double4_mismatch2(double4 p0, double p1) { return max(p1, p0); }
+
+// CHECK-LABEL: define hidden noundef nofpclass(nan inf) float {{.*}}test_max_float_int_mismatch
+// CHECK: [[CONV:%.*]] = sitofp reassoc nnan ninf nsz arcp afn i32 %{{.*}} to float
+// CHECK: [[MAX:%.*]] = call reassoc nnan ninf nsz arcp afn noundef nofpclass(nan inf) float @llvm.maxnum.f32(float {{%.*}}, float [[CONV]])
+// CHECK: ret float [[MAX]]
+float test_max_float_int_mismatch(float p0, int p1) { return max(p0, p1); }
+
+// CHECK-LABEL: define hidden noundef nofpclass(nan inf) float {{.*}}test_max_float_int_mismatch2
+// CHECK: [[CONV:%.*]] = sitofp reassoc nnan ninf nsz arcp afn i32 %{{.*}} to float
+// CHECK: [[MAX:%.*]] = call reassoc nnan ninf nsz arcp afn noundef nofpclass(nan inf) float @llvm.maxnum.f32(float [[CONV]], float %{{.*}})
+// CHECK: ret float [[MAX]]
+float test_max_float_int_mismatch2(float p0, int p1) { return max(p1, p0); }
diff --git a/clang/test/CodeGenHLSL/builtins/min-overloads.hlsl b/clang/test/CodeGenHLSL/builtins/min-overloads.hlsl
index ee3455a07c8e12..be8b45d8b8dfba 100644
--- a/clang/test/CodeGenHLSL/builtins/min-overloads.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/min-overloads.hlsl
@@ -79,3 +79,15 @@ double4 test_min_double4_mismatch(double4 p0, double p1) { return min(p0, p1); }
// CHECK: [[MIN:%.*]] = call reassoc nnan ninf nsz arcp afn noundef nofpclass(nan inf) <4 x double> @llvm.minnum.v4f64(<4 x double> [[CONV1]], <4 x double> %{{.*}})
// CHECK: ret <4 x double> [[MIN]]
double4 test_min_double4_mismatch2(double4 p0, double p1) { return min(p1, p0); }
+
+// CHECK-LABEL: define hidden noundef nofpclass(nan inf) float {{.*}}test_min_float_int_mismatch
+// CHECK: [[CONV:%.*]] = sitofp reassoc nnan ninf nsz arcp afn i32 %{{.*}} to float
+// CHECK: [[MIN:%.*]] = call reassoc nnan ninf nsz arcp afn noundef nofpclass(nan inf) float @llvm.minnum.f32(float {{%.*}}, float [[CONV]])
+// CHECK: ret float [[MIN]]
+float test_min_float_int_mismatch(float p0, int p1) { return min(p0, p1); }
+
+// CHECK-LABEL: define hidden noundef nofpclass(nan inf) float {{.*}}test_min_float_int_mismatch2
+// CHECK: [[CONV:%.*]] = sitofp reassoc nnan ninf nsz arcp afn i32 %{{.*}} to float
+// CHECK: [[MIN:%.*]] = call reassoc nnan ninf nsz arcp afn noundef nofpclass(nan inf) float @llvm.minnum.f32(float [[CONV]], float %{{.*}})
+// CHECK: ret float [[MIN]]
+float test_min_float_int_mismatch2(float p0, int p1) { return min(p1, p0); }
>From 0f6fe38b9c3bd8c468fd428f4e967b1a60bb93be Mon Sep 17 00:00:00 2001
From: Ben Wu <soggysocks206 at gmail.com>
Date: Tue, 6 Oct 2026 14:39:08 -0700
Subject: [PATCH 30/46] [NFC][ARM] Remove unneeded cast (#229504)
`MCConstantExpr::getValue()` returns `int64_t` already.
---
llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 032ba8b0de0411..ab94d8eb21f441 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -9301,8 +9301,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
if (isa<MCConstantExpr>(SubExprVal) &&
Inst.getOperand(0).getReg() != ARM::PC &&
Inst.getOperand(0).getReg() != ARM::SP) {
- int64_t Value =
- (int64_t) (cast<MCConstantExpr>(SubExprVal))->getValue();
+ int64_t Value = (cast<MCConstantExpr>(SubExprVal))->getValue();
bool UseMov = true;
bool MovHasS = true;
if (Inst.getOpcode() == ARM::LDRConstPool) {
>From c1d3899b4364f7d14e900398c95b13ecc820b80e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Susan=20Tan=20=28=E3=82=B9-=E3=82=B6=E3=83=B3=E3=80=80?=
=?UTF-8?q?=E3=82=BF=E3=83=B3=29?= <zujunt at nvidia.com>
Date: Tue, 6 Oct 2026 14:42:11 -0700
Subject: [PATCH 31/46] [flang] Add dummy intent attribute to the function
signature (#229237)
Dummy intent is currently only on fir.declare in the callee body. A
function pass must not read that body, and a func.func private
declaration has no intent.
Lowering now adds `fir.fortran_attrs = #fir.var_attrs<intent_*>` to each
`func.func` argument that has an intent, including caller-created
declarations. Alias analysis reads that attribute, so an intent(out)
dummy is a write without the callee body.
The call no longer looks inside the callee. Intent is read only from the
func.func argument attribute fir.fortran_attrs, and the dummy type comes
from the function type.
---
.../flang/Optimizer/Dialect/FIROpsSupport.h | 46 +++++++++--------
flang/lib/Lower/CallInterface.cpp | 46 +++++++++++++++--
.../lib/Optimizer/Analysis/AliasAnalysis.cpp | 12 +++--
.../AliasAnalysis/modref-call-intent.fir | 49 ++++++++++++++++---
flang/test/Driver/function-attr-readonly.f90 | 10 ++--
flang/test/Lower/CUDA/cuda-gpu-managed.cuf | 2 +-
.../Lower/CUDA/cuf-defined-assignment-rhs.cuf | 2 +-
.../HLFIR/convert-variable-assumed-rank.f90 | 2 +-
.../test/Lower/HLFIR/elemental-array-ops.f90 | 8 +--
.../HLFIR/elemental-polymorphic-merge.f90 | 4 +-
.../Lower/HLFIR/elemental-result-length.f90 | 4 +-
.../Lower/HLFIR/structure-constructor.f90 | 2 +-
.../test/Lower/Intrinsics/c_devptr_eq_ne.f90 | 4 +-
flang/test/Lower/Intrinsics/c_ptr_eq_ne.f90 | 4 +-
flang/test/Lower/Intrinsics/dconjg.f90 | 4 +-
flang/test/Lower/Intrinsics/dimag.f90 | 4 +-
flang/test/Lower/Intrinsics/dreal.f90 | 4 +-
flang/test/Lower/Intrinsics/perror.f90 | 2 +-
flang/test/Lower/Intrinsics/reduce.f90 | 2 +-
flang/test/Lower/OpenACC/acc-declare.f90 | 4 +-
flang/test/Lower/OpenMP/array-bounds.f90 | 4 +-
flang/test/Lower/OpenMP/flush.f90 | 4 +-
.../Lower/OpenMP/optional-argument-map-2.f90 | 4 +-
.../Lower/OpenMP/optional-argument-map-3.f90 | 2 +-
.../Lower/OpenMP/optional-argument-map.f90 | 4 +-
flang/test/Lower/OpenMP/single.f90 | 4 +-
.../wsloop-reduction-array-assumed-shape.f90 | 2 +-
.../test/Lower/OpenMP/wsloop-unstructured.f90 | 8 +--
flang/test/Lower/achar.f90 | 2 +-
flang/test/Lower/arguments.f90 | 4 +-
.../array-elemental-calls-char-dynamic.f90 | 4 +-
flang/test/Lower/call-copy-in-out.f90 | 4 +-
flang/test/Lower/default-initialization.f90 | 2 +-
.../test/Lower/derived-type-finalization.f90 | 2 +-
flang/test/Lower/dispatch.f90 | 4 +-
.../Lower/dummy-argument-attrs-module.f90 | 12 +++++
flang/test/Lower/dummy-argument-attrs-use.f90 | 16 ++++++
flang/test/Lower/dummy-argument-readonly.f90 | 48 +++++++++---------
flang/test/Lower/intentout-deallocate.f90 | 18 +++----
flang/test/Lower/parent-component.f90 | 4 +-
flang/test/Lower/polymorphic-temp.f90 | 24 ++++-----
flang/test/Lower/polymorphic.f90 | 32 ++++++------
.../Lower/repack-arrays-finalized-dummy.f90 | 2 +-
flang/test/Lower/repack-arrays.f90 | 6 +--
flang/test/Lower/select-type-2.f90 | 2 +-
flang/test/Lower/select-type.f90 | 20 ++++----
flang/test/Lower/unsigned-ops.f90 | 4 +-
flang/test/Lower/volatile-string.f90 | 4 +-
flang/test/Lower/volatile1.f90 | 6 +--
49 files changed, 288 insertions(+), 179 deletions(-)
create mode 100644 flang/test/Lower/dummy-argument-attrs-module.f90
create mode 100644 flang/test/Lower/dummy-argument-attrs-use.f90
diff --git a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
index 1fa01bdfc9e889..d5131500ecbbd7 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
+++ b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
@@ -109,6 +109,14 @@ static constexpr llvm::StringRef getReadOnlyAttrName() {
return "fir.read_only";
}
+/// Dummy intent, attached to the corresponding `func.func` argument.
+/// A function pass reads intent from this signature attribute and does not
+/// inspect another function's body. The `fir.` prefix is required: function
+/// arguments may only carry dialect attributes.
+static constexpr llvm::StringRef getFortranAttrsAttrName() {
+ return "fir.fortran_attrs";
+}
+
/// Attribute to mark that a function argument is a character dummy procedure.
/// Character dummy procedure have special ABI constraints.
static constexpr llvm::StringRef getCharacterProcedureDummyAttrName() {
@@ -240,35 +248,31 @@ inline mlir::NamedAttribute getAdaptToByRefAttr(Builder &builder) {
bool isDummyArgument(mlir::Value v);
-/// Intent of dummy argument `argIdx` when `callee`'s body declares it.
-/// Empty when the body is missing or the dummy has no intent attribute.
+/// Intent of function argument `argIdx`.
enum class FortranDummyIntent { In, Out, InOut };
+/// Intent recorded by the `fortran_attrs` attribute of argument `argIdx`.
+/// Empty when that attribute is absent or names no intent. The callee region
+/// is not inspected: a function pass may query another function only through
+/// its signature.
inline std::optional<FortranDummyIntent>
getFortranDummyIntent(mlir::func::FuncOp callee, unsigned argIdx) {
if (!callee || argIdx >= callee.getNumArguments())
return std::nullopt;
- // The dummy's fir.declare uses the block argument as its memref.
- mlir::Value arg = callee.getArgument(argIdx);
- for (mlir::Operation *user : arg.getUsers()) {
- auto decl = mlir::dyn_cast<fir::DeclareOp>(user);
- if (!decl || decl.getMemref() != arg)
- continue;
- auto attrs = decl.getFortranAttrs();
- if (!attrs)
- continue;
- using F = FortranVariableFlagsEnum;
- if (bitEnumContainsAny(*attrs, F::intent_inout) ||
- (bitEnumContainsAny(*attrs, F::intent_in) &&
- bitEnumContainsAny(*attrs, F::intent_out)))
- return FortranDummyIntent::InOut;
- if (bitEnumContainsAny(*attrs, F::intent_out))
- return FortranDummyIntent::Out;
- if (bitEnumContainsAny(*attrs, F::intent_in))
- return FortranDummyIntent::In;
+ auto attrs = callee.getArgAttrOfType<FortranVariableFlagsAttr>(
+ argIdx, getFortranAttrsAttrName());
+ if (!attrs)
+ return std::nullopt;
+ switch (attrs.getFlags()) {
+ case FortranVariableFlagsEnum::intent_in:
+ return FortranDummyIntent::In;
+ case FortranVariableFlagsEnum::intent_out:
+ return FortranDummyIntent::Out;
+ case FortranVariableFlagsEnum::intent_inout:
+ return FortranDummyIntent::InOut;
+ default:
return std::nullopt;
}
- return std::nullopt;
}
template <fir::FortranProcedureFlagsEnum Flag>
diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp
index 2bfe94559cd883..ee6ee38f165f9d 100644
--- a/flang/lib/Lower/CallInterface.cpp
+++ b/flang/lib/Lower/CallInterface.cpp
@@ -252,6 +252,36 @@ static bool dummyArgCanUseLLVMReadonly(
!obj.attrs.test(Attrs::Volatile);
}
+/// Intent of a dummy, recorded on the procedure signature. Both the definition
+/// and a caller-created declaration see this, so intent is available without
+/// the callee body. Other dummy flags stay on their existing attributes.
+static fir::FortranVariableFlagsEnum getDummyIntentFlags(
+ const Fortran::evaluate::characteristics::DummyDataObject &obj) {
+ switch (obj.intent) {
+ case Fortran::common::Intent::In:
+ return fir::FortranVariableFlagsEnum::intent_in;
+ case Fortran::common::Intent::Out:
+ return fir::FortranVariableFlagsEnum::intent_out;
+ case Fortran::common::Intent::InOut:
+ return fir::FortranVariableFlagsEnum::intent_inout;
+ case Fortran::common::Intent::Default:
+ return fir::FortranVariableFlagsEnum::None;
+ }
+ llvm_unreachable("unhandled dummy intent");
+}
+
+static void addFortranVariableFlagsAttr(
+ llvm::SmallVectorImpl<mlir::NamedAttribute> &attrs,
+ mlir::MLIRContext &mlirContext,
+ const Fortran::evaluate::characteristics::DummyDataObject &obj) {
+ fir::FortranVariableFlagsEnum flags = getDummyIntentFlags(obj);
+ if (flags == fir::FortranVariableFlagsEnum::None)
+ return;
+ attrs.emplace_back(
+ mlir::StringAttr::get(&mlirContext, fir::getFortranAttrsAttrName()),
+ fir::FortranVariableFlagsAttr::get(&mlirContext, flags));
+}
+
static Fortran::evaluate::characteristics::DummyArgument
asImplicitArg(Fortran::evaluate::characteristics::DummyArgument &&dummy) {
return Fortran::common::visit(
@@ -1138,15 +1168,18 @@ class Fortran::lower::CallInterfaceImpl {
if (entity) {
if (entity->isPercentVal()) {
mlir::Type type = translateDynamicType(dynamicType);
- addFirOperand(type, nextPassedArgPosition(), Property::Value,
- dummyNameAttr(entity));
+ llvm::SmallVector<mlir::NamedAttribute> attrs = dummyNameAttr(entity);
+ addFortranVariableFlagsAttr(attrs, mlirContext, obj);
+ addFirOperand(type, nextPassedArgPosition(), Property::Value, attrs);
addPassedArg(PassEntityBy::Value, entity, characteristics);
return;
}
if (entity->isPercentRef()) {
mlir::Type refType = getRefType(dynamicType, obj);
+ llvm::SmallVector<mlir::NamedAttribute> attrs = dummyNameAttr(entity);
+ addFortranVariableFlagsAttr(attrs, mlirContext, obj);
addFirOperand(refType, nextPassedArgPosition(), Property::BaseAddress,
- dummyNameAttr(entity));
+ attrs);
addPassedArg(PassEntityBy::BaseAddress, entity, characteristics);
return;
}
@@ -1155,8 +1188,10 @@ class Fortran::lower::CallInterfaceImpl {
if (dynamicType.category() == Fortran::common::TypeCategory::Character) {
mlir::Type boxCharTy =
fir::BoxCharType::get(&mlirContext, dynamicType.kind());
+ llvm::SmallVector<mlir::NamedAttribute> attrs = dummyNameAttr(entity);
+ addFortranVariableFlagsAttr(attrs, mlirContext, obj);
addFirOperand(boxCharTy, nextPassedArgPosition(), Property::BoxChar,
- dummyNameAttr(entity));
+ attrs);
addPassedArg(PassEntityBy::BoxChar, entity, characteristics);
} else {
// non-PDT derived type allowed in implicit interface.
@@ -1168,6 +1203,7 @@ class Fortran::lower::CallInterfaceImpl {
attrs.emplace_back(
mlir::StringAttr::get(&mlirContext, fir::getReadOnlyAttrName()),
mlir::UnitAttr::get(&mlirContext));
+ addFortranVariableFlagsAttr(attrs, mlirContext, obj);
addFirOperand(refType, nextPassedArgPosition(), Property::BaseAddress,
attrs);
addPassedArg(PassEntityBy::BaseAddress, entity, characteristics);
@@ -1232,6 +1268,8 @@ class Fortran::lower::CallInterfaceImpl {
// (see dummyArgCanUseLLVMReadonly).
if (dummyArgCanUseLLVMReadonly(obj))
addMLIRAttr(fir::getReadOnlyAttrName());
+ // Intent only. The unit attributes above stay as they are.
+ addFortranVariableFlagsAttr(attrs, mlirContext, obj);
// TODO: intents that require special care (e.g finalization)
diff --git a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
index 94e3726d6228ac..9e568af7869103 100644
--- a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
+++ b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
@@ -1135,7 +1135,9 @@ ModRefResult AliasAnalysis::getCallModRef(Operation *op, Value var) {
// declared intent is a read, a write, or both. intent(out) is a write for
// a trivial non-pointer, non-allocatable dummy, and a read and a write
// otherwise. An argument with no visible intent stays ModAndRef. The
- // callee is resolved through the cached symbol table.
+ // callee is resolved through the cached symbol table. Intent is a
+ // signature attribute, so a declaration is enough and the callee body is
+ // not inspected.
mlir::func::FuncOp callee;
if (std::optional<mlir::SymbolRefAttr> calleeAttr = call.getCallee()) {
if (const mlir::SymbolTable *symTab = getNearestSymbolTable(call))
@@ -1143,8 +1145,8 @@ ModRefResult AliasAnalysis::getCallModRef(Operation *op, Value var) {
symTab->lookup<mlir::func::FuncOp>(calleeAttr->getLeafReference());
}
auto args = call.getArgs();
- const bool intentsAvailable = callee && !callee.isDeclaration() &&
- args.size() == callee.getNumArguments();
+ const bool intentsAvailable =
+ callee && args.size() == callee.getNumArguments();
ModRefResult modRef = ModRefResult::getNoModRef();
for (auto [idx, arg] : llvm::enumerate(args)) {
if (!fir::conformsWithPassByRef(arg.getType()) || alias(arg, var).isNo())
@@ -1163,7 +1165,9 @@ ModRefResult AliasAnalysis::getCallModRef(Operation *op, Value var) {
// A pure write only for a non-pointer, non-allocatable dummy whose
// element type is trivial. An allocatable is read on entry so it can
// be deallocated, and finalization of a derived type may read it.
- mlir::Type ty = callee.getArgument(idx).getType();
+ // Use the function type: a declaration has no entry block, so
+ // getArgument() is not available.
+ mlir::Type ty = callee.getFunctionType().getInput(idx);
if (fir::isPointerType(ty) || fir::isAllocatableType(ty) ||
!fir::isa_trivial(fir::getFortranElementType(ty)))
return ModRefResult::getModAndRef();
diff --git a/flang/test/Analysis/AliasAnalysis/modref-call-intent.fir b/flang/test/Analysis/AliasAnalysis/modref-call-intent.fir
index 176769c7cdb4f3..54b64763da7227 100644
--- a/flang/test/Analysis/AliasAnalysis/modref-call-intent.fir
+++ b/flang/test/Analysis/AliasAnalysis/modref-call-intent.fir
@@ -2,7 +2,9 @@
// RUN: --mlir-disable-threading %s -o /dev/null 2>&1 | FileCheck %s
// A direct call uses the callee dummy's Fortran intent when the argument
-// aliases the queried variable:
+// aliases the queried variable. Intent is the fortran_attrs attribute on the
+// func.func argument. A fir.declare in the callee body is not consulted, so
+// the same result holds for a declaration with no body.
// intent(in) -> Ref
// intent(out) -> Mod for a trivial non-pointer, non-allocatable dummy
// intent(out) -> ModRef for an allocatable or a derived type
@@ -27,6 +29,12 @@
// CHECK-DAG: alloc_out_call -> other#0: NoModRef{{$}}
// CHECK-DAG: derived_out_call -> derived_out#0: ModRef{{$}}
// CHECK-DAG: derived_out_call -> other#0: NoModRef{{$}}
+// CHECK-DAG: decl_out_call -> decl_out#0: Mod{{$}}
+// CHECK-DAG: decl_out_call -> other#0: NoModRef{{$}}
+// CHECK-DAG: body_only_call -> body_only#0: ModRef{{$}}
+// CHECK-DAG: body_only_call -> other#0: NoModRef{{$}}
+// CHECK-DAG: sig_wins_call -> sig_wins#0: Ref{{$}}
+// CHECK-DAG: sig_wins_call -> other#0: NoModRef{{$}}
func.func @test_call_intent() {
%in_alloca = fir.alloca f32 {bindc_name = "in", uniq_name = "_QFtest_call_intentEin"}
%in = fir.declare %in_alloca uniq_name("_QFtest_call_intentEin") {test.ptr = "in"} : (!fir.ref<f32>) -> !fir.ref<f32>
@@ -52,20 +60,32 @@ func.func @test_call_intent() {
%derived_out_alloca = fir.alloca !fir.type<_QTdt{i:i32}> {bindc_name = "derived_out", uniq_name = "_QFtest_call_intentEderived_out"}
%derived_out = fir.declare %derived_out_alloca uniq_name("_QFtest_call_intentEderived_out") {test.ptr = "derived_out"} : (!fir.ref<!fir.type<_QTdt{i:i32}>>) -> !fir.ref<!fir.type<_QTdt{i:i32}>>
fir.call @_QPfill_derived_out(%derived_out) {test.ptr = "derived_out_call"} : (!fir.ref<!fir.type<_QTdt{i:i32}>>) -> ()
+
+ %decl_out_alloca = fir.alloca f32 {bindc_name = "decl_out", uniq_name = "_QFtest_call_intentEdecl_out"}
+ %decl_out = fir.declare %decl_out_alloca uniq_name("_QFtest_call_intentEdecl_out") {test.ptr = "decl_out"} : (!fir.ref<f32>) -> !fir.ref<f32>
+ fir.call @_QPfill_out_decl(%decl_out) {test.ptr = "decl_out_call"} : (!fir.ref<f32>) -> ()
+
+ %body_only_alloca = fir.alloca f32 {bindc_name = "body_only", uniq_name = "_QFtest_call_intentEbody_only"}
+ %body_only = fir.declare %body_only_alloca uniq_name("_QFtest_call_intentEbody_only") {test.ptr = "body_only"} : (!fir.ref<f32>) -> !fir.ref<f32>
+ fir.call @_QPbody_only_out(%body_only) {test.ptr = "body_only_call"} : (!fir.ref<f32>) -> ()
+
+ %sig_wins_alloca = fir.alloca f32 {bindc_name = "sig_wins", uniq_name = "_QFtest_call_intentEsig_wins"}
+ %sig_wins = fir.declare %sig_wins_alloca uniq_name("_QFtest_call_intentEsig_wins") {test.ptr = "sig_wins"} : (!fir.ref<f32>) -> !fir.ref<f32>
+ fir.call @_QPsig_in_body_out(%sig_wins) {test.ptr = "sig_wins_call"} : (!fir.ref<f32>) -> ()
return
}
-func.func private @_QPread_in(%arg0: !fir.ref<f32>) {
+func.func private @_QPread_in(%arg0: !fir.ref<f32> {fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
%x = fir.declare %arg0 uniq_name("_QFread_inEx") fortran_attrs<intent_in> : (!fir.ref<f32>) -> !fir.ref<f32>
return
}
-func.func private @_QPfill_out(%arg0: !fir.ref<f32>) {
+func.func private @_QPfill_out(%arg0: !fir.ref<f32> {fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
%x = fir.declare %arg0 uniq_name("_QFfill_outEx") fortran_attrs<intent_out> : (!fir.ref<f32>) -> !fir.ref<f32>
return
}
-func.func private @_QPupdate_inout(%arg0: !fir.ref<f32>) {
+func.func private @_QPupdate_inout(%arg0: !fir.ref<f32> {fir.fortran_attrs = #fir.var_attrs<intent_inout>}) {
%x = fir.declare %arg0 uniq_name("_QFupdate_inoutEx") fortran_attrs<intent_inout> : (!fir.ref<f32>) -> !fir.ref<f32>
return
}
@@ -75,19 +95,34 @@ func.func private @_QPno_intent(%arg0: !fir.ref<f32>) {
return
}
-func.func private @_QPscratch(%arg0: !fir.ref<f32>, %arg1: !fir.ref<f32>, %arg2: !fir.ref<f32>) {
+func.func private @_QPscratch(%arg0: !fir.ref<f32> {fir.fortran_attrs = #fir.var_attrs<intent_in>}, %arg1: !fir.ref<f32> {fir.fortran_attrs = #fir.var_attrs<intent_inout>}, %arg2: !fir.ref<f32> {fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
%x = fir.declare %arg0 uniq_name("_QFscratchEx") fortran_attrs<intent_in> : (!fir.ref<f32>) -> !fir.ref<f32>
%a = fir.declare %arg1 uniq_name("_QFscratchEa") fortran_attrs<intent_inout> : (!fir.ref<f32>) -> !fir.ref<f32>
%res = fir.declare %arg2 uniq_name("_QFscratchEres") fortran_attrs<intent_out> : (!fir.ref<f32>) -> !fir.ref<f32>
return
}
-func.func private @_QPfill_alloc_out(%arg0: !fir.ref<!fir.box<!fir.heap<f32>>>) {
+func.func private @_QPfill_alloc_out(%arg0: !fir.ref<!fir.box<!fir.heap<f32>>> {fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
%x = fir.declare %arg0 uniq_name("_QFfill_alloc_outEx") fortran_attrs<allocatable, intent_out> : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> !fir.ref<!fir.box<!fir.heap<f32>>>
return
}
-func.func private @_QPfill_derived_out(%arg0: !fir.ref<!fir.type<_QTdt{i:i32}>>) {
+func.func private @_QPfill_derived_out(%arg0: !fir.ref<!fir.type<_QTdt{i:i32}>> {fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
%x = fir.declare %arg0 uniq_name("_QFfill_derived_outEx") fortran_attrs<intent_out> : (!fir.ref<!fir.type<_QTdt{i:i32}>>) -> !fir.ref<!fir.type<_QTdt{i:i32}>>
return
}
+
+// No body: the signature attribute is the only place intent can live.
+func.func private @_QPfill_out_decl(!fir.ref<f32> {fir.fortran_attrs = #fir.var_attrs<intent_out>})
+
+// Intent on the declare only. It must not be visible across functions.
+func.func private @_QPbody_only_out(%arg0: !fir.ref<f32>) {
+ %x = fir.declare %arg0 uniq_name("_QFbody_only_outEx") fortran_attrs<intent_out> : (!fir.ref<f32>) -> !fir.ref<f32>
+ return
+}
+
+// The signature wins when it disagrees with the declare in the body.
+func.func private @_QPsig_in_body_out(%arg0: !fir.ref<f32> {fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
+ %x = fir.declare %arg0 uniq_name("_QFsig_in_body_outEx") fortran_attrs<intent_out> : (!fir.ref<f32>) -> !fir.ref<f32>
+ return
+}
diff --git a/flang/test/Driver/function-attr-readonly.f90 b/flang/test/Driver/function-attr-readonly.f90
index 120fa8f266da55..5345a12a099757 100644
--- a/flang/test/Driver/function-attr-readonly.f90
+++ b/flang/test/Driver/function-attr-readonly.f90
@@ -31,17 +31,17 @@ subroutine pointer_descriptor_readonly(p)
end module
! O0-LABEL: func.func @_QMreadonly_pipeline_modPreadonly_pipeline(
-! O0-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.read_only, llvm.noalias},
-! O0-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "y", llvm.noalias})
+! O0-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only, llvm.noalias},
+! O0-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "y", fir.fortran_attrs = #fir.var_attrs<intent_inout>, llvm.noalias})
! O1-LABEL: func.func @_QMreadonly_pipeline_modPreadonly_pipeline(
-! O1-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.read_only, llvm.noalias, llvm.nocapture, llvm.readonly},
-! O1-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "y", llvm.noalias, llvm.nocapture})
+! O1-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only, llvm.noalias, llvm.nocapture, llvm.readonly},
+! O1-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "y", fir.fortran_attrs = #fir.var_attrs<intent_inout>, llvm.noalias, llvm.nocapture})
! LLVM-LABEL: define void @_QMreadonly_pipeline_modPreadonly_pipeline(
! LLVM-SAME: ptr {{.*}}readonly{{.*}} %0,
-! O1-LABEL: func.func @_QMreadonly_pipeline_modPpointer_descriptor_readonly(%{{.*}}: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "p", fir.read_only, llvm.nocapture, llvm.readonly})
+! O1-LABEL: func.func @_QMreadonly_pipeline_modPpointer_descriptor_readonly(%{{.*}}: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "p", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only, llvm.nocapture, llvm.readonly})
! LLVM-LABEL: define void @_QMreadonly_pipeline_modPpointer_descriptor_readonly(
! LLVM-SAME: ptr {{.*}}readonly{{.*}} %0
diff --git a/flang/test/Lower/CUDA/cuda-gpu-managed.cuf b/flang/test/Lower/CUDA/cuda-gpu-managed.cuf
index 6fe85f48f9b269..611e96071f4b58 100644
--- a/flang/test/Lower/CUDA/cuda-gpu-managed.cuf
+++ b/flang/test/Lower/CUDA/cuda-gpu-managed.cuf
@@ -179,7 +179,7 @@ subroutine test_dummy_allocatable(arr)
end subroutine
! CHECK-LABEL: func.func @_QPtest_dummy_allocatable(
-! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {cuf.data_attr = #cuf.cuda<managed>, fir.bindc_name = "arr"})
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {cuf.data_attr = #cuf.cuda<managed>, fir.bindc_name = "arr", fir.fortran_attrs = #fir.var_attrs<intent_inout>})
! CHECK: hlfir.declare {{.*}} {{.*}}fortran_attrs<allocatable, intent_inout> data_attr(#cuf.cuda<managed>)
! -----------------------------------------------------------------------------
diff --git a/flang/test/Lower/CUDA/cuf-defined-assignment-rhs.cuf b/flang/test/Lower/CUDA/cuf-defined-assignment-rhs.cuf
index 855e2ef34fa4ff..8ca6e71752f487 100644
--- a/flang/test/Lower/CUDA/cuf-defined-assignment-rhs.cuf
+++ b/flang/test/Lower/CUDA/cuf-defined-assignment-rhs.cuf
@@ -20,7 +20,7 @@ contains
end module
! CHECK-LABEL: func.func @_QMdefined_assignment_device_rhsPassign_t(
-! CHECK-SAME: %[[HOST_T_ARG:.*]]: !fir.ref<!fir.type<_QMdefined_assignment_device_rhsTt{{.*}}> {{.*}}, %[[DEV_X_ARG:.*]]: !fir.box<!fir.array<?xf32>> {cuf.data_attr = #cuf.cuda<device>, fir.bindc_name = "dev_x"})
+! CHECK-SAME: %[[HOST_T_ARG:.*]]: !fir.ref<!fir.type<_QMdefined_assignment_device_rhsTt{{.*}}> {{.*}}, %[[DEV_X_ARG:.*]]: !fir.box<!fir.array<?xf32>> {cuf.data_attr = #cuf.cuda<device>, fir.bindc_name = "dev_x", fir.fortran_attrs = #fir.var_attrs<intent_in>})
! CHECK: %[[DEV_X:.*]]:2 = hlfir.declare %[[DEV_X_ARG]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} {{.*}}data_attr(#cuf.cuda<device>)
! CHECK: %[[HOST_T:.*]]:2 = hlfir.declare %[[HOST_T_ARG]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}}
! CHECK: %[[X_COMPONENT:.*]] = hlfir.designate %[[HOST_T]]#0{"x"}
diff --git a/flang/test/Lower/HLFIR/convert-variable-assumed-rank.f90 b/flang/test/Lower/HLFIR/convert-variable-assumed-rank.f90
index eb344d72098df1..d5f6dc7445f4fb 100644
--- a/flang/test/Lower/HLFIR/convert-variable-assumed-rank.f90
+++ b/flang/test/Lower/HLFIR/convert-variable-assumed-rank.f90
@@ -100,7 +100,7 @@ subroutine test_assumed_length_alloc(x)
! CHECK: }
! CHECK-LABEL: func.func @_QMassumed_rank_testsPtest_intentout(
-! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>> {fir.bindc_name = "x"}) {
+! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] arg {{[0-9]+}} uniq_name("_QMassumed_rank_testsFtest_intentoutEx") fortran_attrs<allocatable, intent_out> : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>)
! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>
diff --git a/flang/test/Lower/HLFIR/elemental-array-ops.f90 b/flang/test/Lower/HLFIR/elemental-array-ops.f90
index 35c33ba28932ca..9a6700811d5a04 100644
--- a/flang/test/Lower/HLFIR/elemental-array-ops.f90
+++ b/flang/test/Lower/HLFIR/elemental-array-ops.f90
@@ -155,8 +155,8 @@ end function callee
l = x==callee(y)
end subroutine char_return
! CHECK-LABEL: func.func @_QPchar_return(
-! CHECK-SAME: %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.char<1,3>>> {fir.bindc_name = "x"},
-! CHECK-SAME: %[[VAL_1:.*]]: !fir.box<!fir.array<?x!fir.char<1,3>>> {fir.bindc_name = "y"}) {
+! CHECK-SAME: %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.char<1,3>>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>},
+! CHECK-SAME: %[[VAL_1:.*]]: !fir.box<!fir.array<?x!fir.char<1,3>>> {fir.bindc_name = "y", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CHECK: %[[VAL_2:.*]] = fir.alloca !fir.char<1,3> <{bindc_name = ".result"}>
! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x!fir.logical<4>>>> <{bindc_name = "l", uniq_name = "_QFchar_returnEl"}>
! CHECK: %[[VAL_4:.*]] = fir.zero_bits !fir.heap<!fir.array<?x!fir.logical<4>>>
@@ -209,7 +209,7 @@ subroutine polymorphic_parenthesis(x, y)
end subroutine polymorphic_parenthesis
! CHECK-LABEL: func.func @_QPpolymorphic_parenthesis(
! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>>> {fir.bindc_name = "x"},
-! CHECK-SAME: %[[VAL_1:.*]]: !fir.class<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>> {fir.bindc_name = "y"}) {
+! CHECK-SAME: %[[VAL_1:.*]]: !fir.class<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>> {fir.bindc_name = "y", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFpolymorphic_parenthesisEx") fortran_attrs<allocatable> : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>>>)
! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFpolymorphic_parenthesisEy") fortran_attrs<intent_in> : (!fir.class<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>, !fir.dscope) -> (!fir.class<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>, !fir.class<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>)
! CHECK: %[[VAL_4:.*]] = arith.constant 0 : index
@@ -233,7 +233,7 @@ subroutine unlimited_polymorphic_parenthesis(x, y)
end subroutine unlimited_polymorphic_parenthesis
! CHECK-LABEL: func.func @_QPunlimited_polymorphic_parenthesis(
! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>> {fir.bindc_name = "x"},
-! CHECK-SAME: %[[VAL_1:.*]]: !fir.class<!fir.array<?xnone>> {fir.bindc_name = "y"}) {
+! CHECK-SAME: %[[VAL_1:.*]]: !fir.class<!fir.array<?xnone>> {fir.bindc_name = "y", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFunlimited_polymorphic_parenthesisEx") fortran_attrs<allocatable> : (!fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>)
! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFunlimited_polymorphic_parenthesisEy") fortran_attrs<intent_in> : (!fir.class<!fir.array<?xnone>>, !fir.dscope) -> (!fir.class<!fir.array<?xnone>>, !fir.class<!fir.array<?xnone>>)
! CHECK: %[[VAL_4:.*]] = arith.constant 0 : index
diff --git a/flang/test/Lower/HLFIR/elemental-polymorphic-merge.f90 b/flang/test/Lower/HLFIR/elemental-polymorphic-merge.f90
index c442ba9e95bc67..902e4700fc43e9 100644
--- a/flang/test/Lower/HLFIR/elemental-polymorphic-merge.f90
+++ b/flang/test/Lower/HLFIR/elemental-polymorphic-merge.f90
@@ -10,8 +10,8 @@ subroutine test_polymorphic_merge(x, y, r, m)
r = merge(x, y, m)
end subroutine test_polymorphic_merge
! CHECK-LABEL: func.func @_QPtest_polymorphic_merge(
-! CHECK-SAME: %[[VAL_0:.*]]: !fir.class<!fir.type<_QFtest_polymorphic_mergeTt>> {fir.bindc_name = "x"},
-! CHECK-SAME: %[[VAL_1:.*]]: !fir.class<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>> {fir.bindc_name = "y"},
+! CHECK-SAME: %[[VAL_0:.*]]: !fir.class<!fir.type<_QFtest_polymorphic_mergeTt>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>},
+! CHECK-SAME: %[[VAL_1:.*]]: !fir.class<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>> {fir.bindc_name = "y", fir.fortran_attrs = #fir.var_attrs<intent_in>},
! CHECK-SAME: %[[VAL_2:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>>> {fir.bindc_name = "r"},
! CHECK-SAME: %[[VAL_3:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "m"}) {
! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFtest_polymorphic_mergeEm") : (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
diff --git a/flang/test/Lower/HLFIR/elemental-result-length.f90 b/flang/test/Lower/HLFIR/elemental-result-length.f90
index 4396388fde1f74..dc270e5e9db26a 100644
--- a/flang/test/Lower/HLFIR/elemental-result-length.f90
+++ b/flang/test/Lower/HLFIR/elemental-result-length.f90
@@ -16,7 +16,7 @@ subroutine sub2(a,b,c)
end subroutine
! CHECK-LABEL: func.func @_QMm1Psub2(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.boxchar<1> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "b"}, %[[ARG2:.*]]: !fir.boxchar<1> {fir.bindc_name = "c"}) {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.boxchar<1> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "b", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[ARG2:.*]]: !fir.boxchar<1> {fir.bindc_name = "c", fir.fortran_attrs = #fir.var_attrs<intent_inout>}) {
! CHECK: %[[UNBOX_ARG0:.*]]:2 = fir.unboxchar %[[ARG0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
! CHECK: %[[A:.*]]:2 = hlfir.declare %[[UNBOX_ARG0]]#0 typeparams %[[UNBOX_ARG0]]#1 dummy_scope %0 {{.*}} uniq_name("_QMm1Fsub2Ea") fortran_attrs<intent_in> : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
! CHECK: %[[UNBOX_ARG1:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
@@ -43,7 +43,7 @@ subroutine sub4(a,b,c)
end subroutine
! CHECK-LABEL: func.func @_QMm1Psub4(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?x!fir.char<1,?>>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.box<!fir.array<?x!fir.char<1,?>>> {fir.bindc_name = "b"}, %[[ARG2:.*]]: !fir.box<!fir.array<?x!fir.char<1,?>>> {fir.bindc_name = "c"}) {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?x!fir.char<1,?>>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[ARG1:.*]]: !fir.box<!fir.array<?x!fir.char<1,?>>> {fir.bindc_name = "b", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[ARG2:.*]]: !fir.box<!fir.array<?x!fir.char<1,?>>> {fir.bindc_name = "c", fir.fortran_attrs = #fir.var_attrs<intent_inout>}) {
! CHECK: %[[A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{.*}} {{.*}} uniq_name("_QMm1Fsub4Ea") fortran_attrs<intent_in> : (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.box<!fir.array<?x!fir.char<1,?>>>)
! CHECK: %[[B:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{.*}} {{.*}} uniq_name("_QMm1Fsub4Eb") fortran_attrs<intent_in> : (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.box<!fir.array<?x!fir.char<1,?>>>)
! CHECK: %[[C:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{.*}} {{.*}} uniq_name("_QMm1Fsub4Ec") fortran_attrs<intent_inout> : (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.box<!fir.array<?x!fir.char<1,?>>>)
diff --git a/flang/test/Lower/HLFIR/structure-constructor.f90 b/flang/test/Lower/HLFIR/structure-constructor.f90
index e08a2fca984608..23b3255896ede9 100644
--- a/flang/test/Lower/HLFIR/structure-constructor.f90
+++ b/flang/test/Lower/HLFIR/structure-constructor.f90
@@ -314,7 +314,7 @@ subroutine test7(n)
x = t7(n)
end subroutine test7
! CHECK-LABEL: func.func @_QPtest7(
-! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "n", fir.read_only}) {
+! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "n", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>
! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFtest7En") fortran_attrs<intent_in> : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}> <{bindc_name = "x", uniq_name = "_QFtest7Ex"}>
diff --git a/flang/test/Lower/Intrinsics/c_devptr_eq_ne.f90 b/flang/test/Lower/Intrinsics/c_devptr_eq_ne.f90
index c69ff5d5e4f617..b6c4fd29cdae97 100644
--- a/flang/test/Lower/Intrinsics/c_devptr_eq_ne.f90
+++ b/flang/test/Lower/Intrinsics/c_devptr_eq_ne.f90
@@ -10,7 +10,7 @@ function test_c_devptr_eq(d1, d2)
end
! CHECK-LABEL: func.func @_QPtest_c_devptr_eq(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>> {fir.bindc_name = "d1"}, %[[ARG1:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>> {fir.bindc_name = "d2"}) -> !fir.logical<4> {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>> {fir.bindc_name = "d1", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[ARG1:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>> {fir.bindc_name = "d2", fir.fortran_attrs = #fir.var_attrs<intent_in>}) -> !fir.logical<4> {
! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFtest_c_devptr_eqEd1") fortran_attrs<intent_in> : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>)
! CHECK: %[[DECL_ARG1:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFtest_c_devptr_eqEd2") fortran_attrs<intent_in> : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>)
! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.logical<4> <{bindc_name = "test_c_devptr_eq", uniq_name = "_QFtest_c_devptr_eqEtest_c_devptr_eq"}>
@@ -38,7 +38,7 @@ function test_c_devptr_ne(d1, d2)
end
! CHECK-LABEL: func.func @_QPtest_c_devptr_ne(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>> {fir.bindc_name = "d1"}, %[[ARG1:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>> {fir.bindc_name = "d2"}) -> !fir.logical<4> {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>> {fir.bindc_name = "d1", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[ARG1:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>> {fir.bindc_name = "d2", fir.fortran_attrs = #fir.var_attrs<intent_in>}) -> !fir.logical<4> {
! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFtest_c_devptr_neEd1") fortran_attrs<intent_in> : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>)
! CHECK: %[[DECL_ARG1:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFtest_c_devptr_neEd2") fortran_attrs<intent_in> : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>>)
! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.logical<4> <{bindc_name = "test_c_devptr_ne", uniq_name = "_QFtest_c_devptr_neEtest_c_devptr_ne"}>
diff --git a/flang/test/Lower/Intrinsics/c_ptr_eq_ne.f90 b/flang/test/Lower/Intrinsics/c_ptr_eq_ne.f90
index 2e64445904106a..ace83c02341661 100644
--- a/flang/test/Lower/Intrinsics/c_ptr_eq_ne.f90
+++ b/flang/test/Lower/Intrinsics/c_ptr_eq_ne.f90
@@ -9,7 +9,7 @@ function test_c_ptr_eq(ptr1, ptr2)
end
! CHECK-LABEL: func.func @_QPtest_c_ptr_eq(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "ptr1"}, %[[ARG1:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "ptr2"}) -> !fir.logical<4> {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "ptr1", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[ARG1:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "ptr2", fir.fortran_attrs = #fir.var_attrs<intent_in>}) -> !fir.logical<4> {
! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFtest_c_ptr_eqEptr1") fortran_attrs<intent_in> : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
! CHECK: %[[DECL_ARG1:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFtest_c_ptr_eqEptr2") fortran_attrs<intent_in> : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.logical<4> <{bindc_name = "test_c_ptr_eq", uniq_name = "_QFtest_c_ptr_eqEtest_c_ptr_eq"}>
@@ -34,7 +34,7 @@ function test_c_ptr_ne(ptr1, ptr2)
end
! CHECK-LABEL: func.func @_QPtest_c_ptr_ne(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "ptr1"}, %[[ARG1:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "ptr2"}) -> !fir.logical<4> {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "ptr1", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[ARG1:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "ptr2", fir.fortran_attrs = #fir.var_attrs<intent_in>}) -> !fir.logical<4> {
! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFtest_c_ptr_neEptr1") fortran_attrs<intent_in> : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
! CHECK: %[[DECL_ARG1:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFtest_c_ptr_neEptr2") fortran_attrs<intent_in> : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.logical<4> <{bindc_name = "test_c_ptr_ne", uniq_name = "_QFtest_c_ptr_neEtest_c_ptr_ne"}>
diff --git a/flang/test/Lower/Intrinsics/dconjg.f90 b/flang/test/Lower/Intrinsics/dconjg.f90
index b223259a39feb0..8ab2d0a063764f 100644
--- a/flang/test/Lower/Intrinsics/dconjg.f90
+++ b/flang/test/Lower/Intrinsics/dconjg.f90
@@ -5,8 +5,8 @@ subroutine test_dconjg(r, c)
complex(8), intent(in) :: c
! CHECK-LABEL: func @_QPtest_dconjg(
-! CHECK-SAME: %[[ARG_0:.*]]: !fir.ref<complex<f64>> {fir.bindc_name = "r"},
-! CHECK-SAME: %[[ARG_1:.*]]: !fir.ref<complex<f64>> {fir.bindc_name = "c", fir.read_only}) {
+! CHECK-SAME: %[[ARG_0:.*]]: !fir.ref<complex<f64>> {fir.bindc_name = "r", fir.fortran_attrs = #fir.var_attrs<intent_out>},
+! CHECK-SAME: %[[ARG_1:.*]]: !fir.ref<complex<f64>> {fir.bindc_name = "c", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
! CHECK: %[[DS:.*]] = fir.dummy_scope : !fir.dscope
! CHECK: %[[C_DECL:.*]]:2 = hlfir.declare %[[ARG_1]] dummy_scope %[[DS]] {{.*}}
! CHECK: %[[R_DECL:.*]]:2 = hlfir.declare %[[ARG_0]] dummy_scope %[[DS]] {{.*}}
diff --git a/flang/test/Lower/Intrinsics/dimag.f90 b/flang/test/Lower/Intrinsics/dimag.f90
index 7826d441b2e9c4..be3eac4b21bb6b 100644
--- a/flang/test/Lower/Intrinsics/dimag.f90
+++ b/flang/test/Lower/Intrinsics/dimag.f90
@@ -5,8 +5,8 @@ subroutine test_dimag(r, c)
complex(8), intent(in) :: c
! CHECK-LABEL: func @_QPtest_dimag(
-! CHECK-SAME: %[[ARG_0:.*]]: !fir.ref<f64> {fir.bindc_name = "r"},
-! CHECK-SAME: %[[ARG_1:.*]]: !fir.ref<complex<f64>> {fir.bindc_name = "c", fir.read_only}) {
+! CHECK-SAME: %[[ARG_0:.*]]: !fir.ref<f64> {fir.bindc_name = "r", fir.fortran_attrs = #fir.var_attrs<intent_out>},
+! CHECK-SAME: %[[ARG_1:.*]]: !fir.ref<complex<f64>> {fir.bindc_name = "c", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
! CHECK: %[[DS:.*]] = fir.dummy_scope : !fir.dscope
! CHECK: %[[C_DECL:.*]]:2 = hlfir.declare %[[ARG_1]] dummy_scope %[[DS]] {{.*}}
! CHECK: %[[R_DECL:.*]]:2 = hlfir.declare %[[ARG_0]] dummy_scope %[[DS]] {{.*}}
diff --git a/flang/test/Lower/Intrinsics/dreal.f90 b/flang/test/Lower/Intrinsics/dreal.f90
index 3f72aa7d3d630e..76f7abd49ed1fe 100644
--- a/flang/test/Lower/Intrinsics/dreal.f90
+++ b/flang/test/Lower/Intrinsics/dreal.f90
@@ -5,8 +5,8 @@ subroutine test_dreal(r, c)
complex(8), intent(in) :: c
! CHECK-LABEL: func.func @_QPtest_dreal(
-! CHECK-SAME: %[[ARG_0:.*]]: !fir.ref<f64> {fir.bindc_name = "r"},
-! CHECK-SAME: %[[ARG_1:.*]]: !fir.ref<complex<f64>> {fir.bindc_name = "c", fir.read_only}) {
+! CHECK-SAME: %[[ARG_0:.*]]: !fir.ref<f64> {fir.bindc_name = "r", fir.fortran_attrs = #fir.var_attrs<intent_out>},
+! CHECK-SAME: %[[ARG_1:.*]]: !fir.ref<complex<f64>> {fir.bindc_name = "c", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
! CHECK: %[[DS:.*]] = fir.dummy_scope : !fir.dscope
! CHECK: %[[C:.*]]:2 = hlfir.declare %[[ARG_1]] dummy_scope %[[DS]]
! CHECK: %[[R:.*]]:2 = hlfir.declare %[[ARG_0]] dummy_scope %[[DS]]
diff --git a/flang/test/Lower/Intrinsics/perror.f90 b/flang/test/Lower/Intrinsics/perror.f90
index f67d92640d5657..da86d7d6eaf588 100644
--- a/flang/test/Lower/Intrinsics/perror.f90
+++ b/flang/test/Lower/Intrinsics/perror.f90
@@ -35,7 +35,7 @@ subroutine test_perror()
end subroutine test_perror
! CHECK-LABEL: func @_QPtest_perror_unknown_length(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.boxchar<1> {fir.bindc_name = "str"}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.boxchar<1> {fir.bindc_name = "str", fir.fortran_attrs = #fir.var_attrs<intent_in>}
subroutine test_perror_unknown_length(str)
implicit none
character(len=*), intent(in) :: str
diff --git a/flang/test/Lower/Intrinsics/reduce.f90 b/flang/test/Lower/Intrinsics/reduce.f90
index f5e1125316808a..20c705153b192a 100644
--- a/flang/test/Lower/Intrinsics/reduce.f90
+++ b/flang/test/Lower/Intrinsics/reduce.f90
@@ -64,7 +64,7 @@ subroutine integer1(a, id, d1, d2)
end subroutine
! CHECK-LABEL: func.func @_QMreduce_modPinteger1(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi8>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<i8> {fir.bindc_name = "id"}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi8>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[ARG1:.*]]: !fir.ref<i8> {fir.bindc_name = "id"}
! CHECK: %[[A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{.*}} uniq_name("_QMreduce_modFinteger1Ea") fortran_attrs<intent_in> : (!fir.box<!fir.array<?xi8>>, !fir.dscope) -> (!fir.box<!fir.array<?xi8>>, !fir.box<!fir.array<?xi8>>)
! CHECK: %[[ID:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{.*}} uniq_name("_QMreduce_modFinteger1Eid") : (!fir.ref<i8>, !fir.dscope) -> (!fir.ref<i8>, !fir.ref<i8>)
! CHECK: %[[ALLOC_RES:.*]] = fir.alloca i8 <{bindc_name = "res", uniq_name = "_QMreduce_modFinteger1Eres"}>
diff --git a/flang/test/Lower/OpenACC/acc-declare.f90 b/flang/test/Lower/OpenACC/acc-declare.f90
index 4b970e8cdd6087..4aa7203716d6ea 100644
--- a/flang/test/Lower/OpenACC/acc-declare.f90
+++ b/flang/test/Lower/OpenACC/acc-declare.f90
@@ -74,8 +74,8 @@ subroutine acc_declare_present_host_and_comp(host, coeffs)
end subroutine
! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_present_host_and_comp(
-! CHECK-SAME: %[[ARGHOST:.*]]: !fir.ref<!fir.type<_QMacc_declare{{.*}}Tdt_box{{.*}}>> {fir.bindc_name = "host"},
-! CHECK-SAME: %[[ARGCOEFF:.*]]: !fir.ref<!fir.array<10xf32>> {fir.bindc_name = "coeffs"})
+! CHECK-SAME: %[[ARGHOST:.*]]: !fir.ref<!fir.type<_QMacc_declare{{.*}}Tdt_box{{.*}}>> {fir.bindc_name = "host", fir.fortran_attrs = #fir.var_attrs<intent_in>},
+! CHECK-SAME: %[[ARGCOEFF:.*]]: !fir.ref<!fir.array<10xf32>> {fir.bindc_name = "coeffs", fir.fortran_attrs = #fir.var_attrs<intent_in>})
! CHECK-DAG: acc.present{{.*}}name("host")
! CHECK-DAG: acc.present{{.*}}name("host%buf")
! CHECK-DAG: acc.present{{.*}}name("coeffs")
diff --git a/flang/test/Lower/OpenMP/array-bounds.f90 b/flang/test/Lower/OpenMP/array-bounds.f90
index 84ae7e5b1203f4..12fe0c9d0b0a42 100644
--- a/flang/test/Lower/OpenMP/array-bounds.f90
+++ b/flang/test/Lower/OpenMP/array-bounds.f90
@@ -39,7 +39,7 @@ module assumed_array_routines
contains
!HOST-LABEL: func.func @_QMassumed_array_routinesPassumed_shape_array(
-!HOST-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "arr_read_write"}) {
+!HOST-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "arr_read_write", fir.fortran_attrs = #fir.var_attrs<intent_inout>}) {
!HOST: %[[INTERMEDIATE_ALLOCA:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
!HOST: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QMassumed_array_routinesFassumed_shape_arrayEarr_read_write") fortran_attrs<intent_inout> : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
!HOST: %[[C0:.*]] = arith.constant 1 : index
@@ -67,7 +67,7 @@ end subroutine assumed_shape_array
!HOST-LABEL: func.func @_QMassumed_array_routinesPassumed_size_array(
-!HOST-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?xi32>> {fir.bindc_name = "arr_read_write"}) {
+!HOST-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?xi32>> {fir.bindc_name = "arr_read_write", fir.fortran_attrs = #fir.var_attrs<intent_inout>}) {
!HOST: %[[ARG0_SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
!HOST: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]](%[[ARG0_SHAPE]]) dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QMassumed_array_routinesFassumed_size_arrayEarr_read_write") fortran_attrs<intent_inout> : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>)
!HOST: %[[ALLOCA:.*]] = fir.alloca i32 <{bindc_name = "i", uniq_name = "_QMassumed_array_routinesFassumed_size_arrayEi"}>
diff --git a/flang/test/Lower/OpenMP/flush.f90 b/flang/test/Lower/OpenMP/flush.f90
index baa52ee09d79b9..16013c48ca6476 100644
--- a/flang/test/Lower/OpenMP/flush.f90
+++ b/flang/test/Lower/OpenMP/flush.f90
@@ -3,7 +3,7 @@
!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
!CHECK-LABEL: func.func @_QPflush_standalone
-!CHECK-SAME: %[[ARG_A:.*]]: !fir.ref<i32> {fir.bindc_name = "a"}, %[[ARG_B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"}, %[[ARG_C:.*]]: !fir.ref<i32> {fir.bindc_name = "c"})
+!CHECK-SAME: %[[ARG_A:.*]]: !fir.ref<i32> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_inout>}, %[[ARG_B:.*]]: !fir.ref<i32> {fir.bindc_name = "b", fir.fortran_attrs = #fir.var_attrs<intent_inout>}, %[[ARG_C:.*]]: !fir.ref<i32> {fir.bindc_name = "c", fir.fortran_attrs = #fir.var_attrs<intent_inout>})
subroutine flush_standalone(a, b, c)
integer, intent(inout) :: a, b, c
@@ -18,7 +18,7 @@ subroutine flush_standalone(a, b, c)
end subroutine flush_standalone
!CHECK-LABEL: func.func @_QPflush_parallel
-!CHECK-SAME: %[[ARG_A:.*]]: !fir.ref<i32> {fir.bindc_name = "a"}, %[[ARG_B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"}, %[[ARG_C:.*]]: !fir.ref<i32> {fir.bindc_name = "c"})
+!CHECK-SAME: %[[ARG_A:.*]]: !fir.ref<i32> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_inout>}, %[[ARG_B:.*]]: !fir.ref<i32> {fir.bindc_name = "b", fir.fortran_attrs = #fir.var_attrs<intent_inout>}, %[[ARG_C:.*]]: !fir.ref<i32> {fir.bindc_name = "c", fir.fortran_attrs = #fir.var_attrs<intent_inout>})
subroutine flush_parallel(a, b, c)
integer, intent(inout) :: a, b, c
!CHECK: %[[A:.*]]:2 = hlfir.declare %[[ARG_A]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFflush_parallelEa") fortran_attrs<intent_inout> : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenMP/optional-argument-map-2.f90 b/flang/test/Lower/OpenMP/optional-argument-map-2.f90
index 9aac46632c0694..d904b0fbefed49 100644
--- a/flang/test/Lower/OpenMP/optional-argument-map-2.f90
+++ b/flang/test/Lower/OpenMP/optional-argument-map-2.f90
@@ -25,7 +25,7 @@ end subroutine routine_boxchar
end module mod
! CHECK-LABEL: func.func @_QMmodProutine_box(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "a", fir.optional}) {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_inout>, fir.optional}) {
! CHECK: %[[VAL_0:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>>
! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_1]] arg {{[0-9]+}} uniq_name("_QMmodFroutine_boxEa") fortran_attrs<allocatable, intent_inout, optional> : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
@@ -54,7 +54,7 @@ end module mod
! CHECK: }
! CHECK-LABEL: func.func @_QMmodProutine_boxchar(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.boxchar<1> {fir.bindc_name = "a", fir.optional}) {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.boxchar<1> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.optional}) {
! CHECK: %[[VAL_0:.*]] = fir.alloca !fir.boxchar<1>
! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
! CHECK: %[[VAL_2:.*]]:2 = fir.unboxchar %[[ARG0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
diff --git a/flang/test/Lower/OpenMP/optional-argument-map-3.f90 b/flang/test/Lower/OpenMP/optional-argument-map-3.f90
index e0b762f615d06f..f0859b0cd5b846 100644
--- a/flang/test/Lower/OpenMP/optional-argument-map-3.f90
+++ b/flang/test/Lower/OpenMP/optional-argument-map-3.f90
@@ -23,7 +23,7 @@ end subroutine foo
end module
! CHECK-LABEL: func.func @{{.*}}(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "dt"},
+! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "dt", fir.fortran_attrs = #fir.var_attrs<intent_inout>},
! CHECK: %[[VAL_0:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope {{.*}}
! CHECK: fir.if %{{.*}} {
diff --git a/flang/test/Lower/OpenMP/optional-argument-map.f90 b/flang/test/Lower/OpenMP/optional-argument-map.f90
index 6458d47526a779..52a44802bfaf00 100644
--- a/flang/test/Lower/OpenMP/optional-argument-map.f90
+++ b/flang/test/Lower/OpenMP/optional-argument-map.f90
@@ -15,8 +15,8 @@ end subroutine test
end module foo
! CHECK-LABEL: func.func @_QMfooPtest(
-! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.read_only},
-! CHECK-SAME: %[[VAL_1:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a", fir.optional}) {
+! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only},
+! CHECK-SAME: %[[VAL_1:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_inout>, fir.optional}) {
! CHECK: %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.array<?xf32>>
! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{.*}} uniq_name("_QMfooFtestEa") fortran_attrs<intent_inout, optional> : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
! CHECK: %{{.*}} = fir.is_present %{{.*}}#1 : (!fir.box<!fir.array<?xf32>>) -> i1
diff --git a/flang/test/Lower/OpenMP/single.f90 b/flang/test/Lower/OpenMP/single.f90
index 1b1a8894f54136..d4a54c4bd21e61 100644
--- a/flang/test/Lower/OpenMP/single.f90
+++ b/flang/test/Lower/OpenMP/single.f90
@@ -8,7 +8,7 @@
!===============================================================================
!CHECK-LABEL: func @_QPomp_single
-!CHECK-SAME: (%[[X:.*]]: !fir.ref<i32> {fir.bindc_name = "x"})
+!CHECK-SAME: (%[[X:.*]]: !fir.ref<i32> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_inout>})
subroutine omp_single(x)
integer, intent(inout) :: x
!CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFomp_singleEx") fortran_attrs<intent_inout> : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -31,7 +31,7 @@ end subroutine omp_single
!===============================================================================
!CHECK-LABEL: func @_QPomp_single_nowait
-!CHECK-SAME: (%[[X:.*]]: !fir.ref<i32> {fir.bindc_name = "x"})
+!CHECK-SAME: (%[[X:.*]]: !fir.ref<i32> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_inout>})
subroutine omp_single_nowait(x)
integer, intent(inout) :: x
!CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFomp_single_nowaitEx") fortran_attrs<intent_inout> : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
index dcdc286a138f3e..e74a0a9f2deb1a 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
@@ -74,7 +74,7 @@ subroutine reduce(r)
! CHECK: }
! CHECK-LABEL: func.func private @_QFPreduce(
-! CHECK-SAME: %[[VAL_0:.*]]: !fir.box<!fir.array<?xf64>> {fir.bindc_name = "r"}) attributes {{.*}} {
+! CHECK-SAME: %[[VAL_0:.*]]: !fir.box<!fir.array<?xf64>> {fir.bindc_name = "r", fir.fortran_attrs = #fir.var_attrs<intent_inout>}) attributes {{.*}} {
! CHECK: %[[VAL_1:.*]] = fir.address_of(@_QFFreduceEi) : !fir.ref<i32>
! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] uniq_name("_QFFreduceEi") : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} arg {{[0-9]+}} uniq_name("_QFFreduceEr") fortran_attrs<{{.*}}> : (!fir.box<!fir.array<?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?xf64>>, !fir.box<!fir.array<?xf64>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-unstructured.f90 b/flang/test/Lower/OpenMP/wsloop-unstructured.f90
index f4a0777a603be7..9fbdf48dcf16c5 100644
--- a/flang/test/Lower/OpenMP/wsloop-unstructured.f90
+++ b/flang/test/Lower/OpenMP/wsloop-unstructured.f90
@@ -24,10 +24,10 @@ end subroutine sub
! that all blocks are terminated
! CHECK-LABEL: func.func @_QPsub(
-! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "imax", fir.read_only},
-! CHECK-SAME: %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "jmax", fir.read_only},
-! CHECK-SAME: %[[VAL_2:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "x"},
-! CHECK-SAME: %[[VAL_3:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "y"}) {
+! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "imax", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only},
+! CHECK-SAME: %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "jmax", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only},
+! CHECK-SAME: %[[VAL_2:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>},
+! CHECK-SAME: %[[VAL_3:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "y", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! [...]
! CHECK: omp.wsloop private({{.*}}) {
! CHECK-NEXT: omp.loop_nest (%[[VAL_53:.*]], %[[VAL_54:.*]]) : i32 = ({{.*}}) to ({{.*}}) inclusive step ({{.*}}) {
diff --git a/flang/test/Lower/achar.f90 b/flang/test/Lower/achar.f90
index 3d9be967f7e0b9..c916df45e80d4e 100644
--- a/flang/test/Lower/achar.f90
+++ b/flang/test/Lower/achar.f90
@@ -12,7 +12,7 @@ subroutine achar_test1(a)
end subroutine achar_test1
! CHECK-LABEL: func.func @_QPachar_test1(
-! CHECK-SAME: %[[ARG:.*]]: !fir.ref<i32> {fir.bindc_name = "a", fir.read_only}) {
+! CHECK-SAME: %[[ARG:.*]]: !fir.ref<i32> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
! CHECK: %[[TMP:.*]] = fir.alloca !fir.char<1>
! CHECK: %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
! CHECK: %[[A:.*]]:2 = hlfir.declare %[[ARG]] dummy_scope %[[DSCOPE]] arg 1 uniq_name("_QFachar_test1Ea") fortran_attrs<intent_in>
diff --git a/flang/test/Lower/arguments.f90 b/flang/test/Lower/arguments.f90
index 9667494f0f0113..870a8807e5c0f1 100644
--- a/flang/test/Lower/arguments.f90
+++ b/flang/test/Lower/arguments.f90
@@ -7,7 +7,7 @@ subroutine sub1(a, b)
! Check that arguments are correctly set and no local allocation is happening.
! CHECK-LABEL: func @_QPsub1(
-! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "a", fir.read_only}, %{{.*}}: !fir.ref<!fir.logical<4>> {fir.bindc_name = "b"})
+! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}, %{{.*}}: !fir.ref<!fir.logical<4>> {fir.bindc_name = "b"})
! CHECK-NOT: fir.alloc
! CHECK: return
@@ -31,7 +31,7 @@ integer function fct1(a, b)
end
! CHECK-LABEL: func @_QPfct1(
-! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "a", fir.read_only}, %{{.*}}: !fir.ref<!fir.logical<4>> {fir.bindc_name = "b"}) -> i32
+! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}, %{{.*}}: !fir.ref<!fir.logical<4>> {fir.bindc_name = "b"}) -> i32
real function fct2(i)
integer :: i(2, 5)
diff --git a/flang/test/Lower/array-elemental-calls-char-dynamic.f90 b/flang/test/Lower/array-elemental-calls-char-dynamic.f90
index b9a221b29ec64d..3eeb3873a1cfb7 100644
--- a/flang/test/Lower/array-elemental-calls-char-dynamic.f90
+++ b/flang/test/Lower/array-elemental-calls-char-dynamic.f90
@@ -186,8 +186,8 @@ elemental function f_poly(p1, p2)
end subroutine
! CHECK-LABEL: func.func @_QPtest_polymorphic(
! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?x!fir.char<1,?>>> {fir.bindc_name = "res"},
-! CHECK-SAME: %[[ARG1:.*]]: !fir.class<!fir.array<?x!fir.type<_QFtest_polymorphicTt>>> {fir.bindc_name = "p1"},
-! CHECK-SAME: %[[ARG2:.*]]: !fir.class<!fir.array<?x!fir.type<_QFtest_polymorphicTt>>> {fir.bindc_name = "p2"}) {
+! CHECK-SAME: %[[ARG1:.*]]: !fir.class<!fir.array<?x!fir.type<_QFtest_polymorphicTt>>> {fir.bindc_name = "p1", fir.fortran_attrs = #fir.var_attrs<intent_in>},
+! CHECK-SAME: %[[ARG2:.*]]: !fir.class<!fir.array<?x!fir.type<_QFtest_polymorphicTt>>> {fir.bindc_name = "p2", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CHECK: %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %[[VAL_0]] arg {{[0-9]+}} uniq_name("_QFtest_polymorphicEp1") fortran_attrs<intent_in> : (!fir.class<!fir.array<?x!fir.type<_QFtest_polymorphicTt>>>, !fir.dscope) -> (!fir.class<!fir.array<?x!fir.type<_QFtest_polymorphicTt>>>, !fir.class<!fir.array<?x!fir.type<_QFtest_polymorphicTt>>>)
! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %[[VAL_0]] arg {{[0-9]+}} uniq_name("_QFtest_polymorphicEp2") fortran_attrs<intent_in> : (!fir.class<!fir.array<?x!fir.type<_QFtest_polymorphicTt>>>, !fir.dscope) -> (!fir.class<!fir.array<?x!fir.type<_QFtest_polymorphicTt>>>, !fir.class<!fir.array<?x!fir.type<_QFtest_polymorphicTt>>>)
diff --git a/flang/test/Lower/call-copy-in-out.f90 b/flang/test/Lower/call-copy-in-out.f90
index 06d6af28ee55e3..df9e77ff3ac6ed 100644
--- a/flang/test/Lower/call-copy-in-out.f90
+++ b/flang/test/Lower/call-copy-in-out.f90
@@ -104,7 +104,7 @@ subroutine test_actual_arg_intent_in(x)
! at the inner call site, so the inner procedure still generates copy-back.
! The outer array must therefore not be marked fir.read_only.
! CHECK-LABEL: func.func @_QPtest_forwarded_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<4xf32>> {fir.bindc_name = "x"}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<4xf32>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
subroutine test_forwarded_intent_in(x)
real, intent(in) :: x(4)
call test_forwarded_without_intent(x)
@@ -142,7 +142,7 @@ subroutine test_actual_arg_intent_in_ptr_component(x)
! the target's contents, so the callee may define the target and copy-out is
! required.
! CHECK-LABEL: func.func @_QPtest_actual_intent_in_pointer(
-! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "pi", fir.read_only}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "pi", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
subroutine test_actual_intent_in_pointer(pi)
integer, intent(in), pointer :: pi(:)
! CHECK: hlfir.copy_in
diff --git a/flang/test/Lower/default-initialization.f90 b/flang/test/Lower/default-initialization.f90
index 9b0fcf115ccd3e..093310a80b5284 100644
--- a/flang/test/Lower/default-initialization.f90
+++ b/flang/test/Lower/default-initialization.f90
@@ -78,7 +78,7 @@ subroutine intent_out(x)
! Test that optional intent(out) are default initialized only when
! present.
! CHECK-LABEL: func @_QMtest_dinitPintent_out_optional(
- ! CHECK-SAME: %[[arg0:.*]]: !fir.ref<!fir.type<_QMtest_dinitTt{{(,sequence)?}}{i:i32}>> {fir.bindc_name = "x", fir.optional})
+ ! CHECK-SAME: %[[arg0:.*]]: !fir.ref<!fir.type<_QMtest_dinitTt{{(,sequence)?}}{i:i32}>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_out>, fir.optional})
subroutine intent_out_optional(x)
! CHECK: %[[x:.*]]:2 = hlfir.declare %[[arg0]] {{.*}} uniq_name("_QMtest_dinitFintent_out_optionalEx") fortran_attrs<intent_out, optional>
! CHECK: %[[isPresent:.*]] = fir.is_present %[[x]]#0 : (!fir.ref<!fir.type<_QMtest_dinitTt{{(,sequence)?}}{i:i32}>>) -> i1
diff --git a/flang/test/Lower/derived-type-finalization.f90 b/flang/test/Lower/derived-type-finalization.f90
index 8fbed8fbbf1f11..652cbb60800b31 100644
--- a/flang/test/Lower/derived-type-finalization.f90
+++ b/flang/test/Lower/derived-type-finalization.f90
@@ -152,7 +152,7 @@ subroutine test_finalize_intent_out(t)
end subroutine
! CHECK-LABEL: func.func @_QMderived_type_finalizationPtest_finalize_intent_out(
-! CHECK-SAME: %[[T:.*]]: !fir.ref<!fir.type<_QMderived_type_finalizationTt1{a:i32}>> {fir.bindc_name = "t"}) {
+! CHECK-SAME: %[[T:.*]]: !fir.ref<!fir.type<_QMderived_type_finalizationTt1{a:i32}>> {fir.bindc_name = "t", fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
! CHECK: %[[T_DECL:.*]]:2 = hlfir.declare %[[T]]
! CHECK: fir.call @_FortranADestroy(%{{.*}})
! CHECK: return
diff --git a/flang/test/Lower/dispatch.f90 b/flang/test/Lower/dispatch.f90
index 72505edc87d57d..d5a98b6801922c 100644
--- a/flang/test/Lower/dispatch.f90
+++ b/flang/test/Lower/dispatch.f90
@@ -107,7 +107,7 @@ subroutine p1_proc4_arg1(i, this)
integer, intent(in) :: i
class(p1) :: this
end subroutine
- ! CHECK-LABEL: func.func @_QMcall_dispatchPp1_proc4_arg1(%{{.*}}: !fir.ref<i32> {fir.bindc_name = "i", fir.read_only}, %{{.*}}: !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>> {fir.bindc_name = "this"})
+ ! CHECK-LABEL: func.func @_QMcall_dispatchPp1_proc4_arg1(%{{.*}}: !fir.ref<i32> {fir.bindc_name = "i", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}, %{{.*}}: !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>> {fir.bindc_name = "this"})
subroutine tbp_nopass()
end subroutine
@@ -127,7 +127,7 @@ subroutine tbp_pass_arg1(i, this)
integer, intent(in) :: i
class(p1) :: this
end subroutine
- ! CHECK-LABEL: func.func @_QMcall_dispatchPtbp_pass_arg1(%{{.*}}: !fir.ref<i32> {fir.bindc_name = "i", fir.read_only}, %{{.*}}: !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>> {fir.bindc_name = "this"})
+ ! CHECK-LABEL: func.func @_QMcall_dispatchPtbp_pass_arg1(%{{.*}}: !fir.ref<i32> {fir.bindc_name = "i", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}, %{{.*}}: !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>> {fir.bindc_name = "this"})
subroutine check_dispatch(p)
class(p1) :: p
diff --git a/flang/test/Lower/dummy-argument-attrs-module.f90 b/flang/test/Lower/dummy-argument-attrs-module.f90
new file mode 100644
index 00000000000000..95da12008a4c19
--- /dev/null
+++ b/flang/test/Lower/dummy-argument-attrs-module.f90
@@ -0,0 +1,12 @@
+! The defining compilation records dummy intent on the func.func arguments.
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s
+
+! CHECK: func.func @_QMmPtwice(%{{.*}}: !fir.ref<f64> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}, %{{.*}}: !fir.ref<f64> {fir.bindc_name = "y", fir.fortran_attrs = #fir.var_attrs<intent_out>})
+module m
+contains
+ subroutine twice(x, y)
+ double precision, intent(in) :: x
+ double precision, intent(out) :: y
+ y = 2 * x
+ end subroutine twice
+end module m
diff --git a/flang/test/Lower/dummy-argument-attrs-use.f90 b/flang/test/Lower/dummy-argument-attrs-use.f90
new file mode 100644
index 00000000000000..d83b51998a5bcc
--- /dev/null
+++ b/flang/test/Lower/dummy-argument-attrs-use.f90
@@ -0,0 +1,16 @@
+! The caller only has the module interface. Intent must still appear on the
+! private func.func declaration, with no callee body to inspect.
+!
+! RUN: rm -fr %t && mkdir -p %t && cd %t
+! RUN: bbc -emit-hlfir %S/dummy-argument-attrs-module.f90
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s
+
+program p
+ use m
+ implicit none
+ double precision :: a, b
+ a = 1.0d0
+ call twice(a, b)
+end program p
+
+! CHECK: func.func private @_QMmPtwice(!fir.ref<f64> {fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}, !fir.ref<f64> {fir.fortran_attrs = #fir.var_attrs<intent_out>})
diff --git a/flang/test/Lower/dummy-argument-readonly.f90 b/flang/test/Lower/dummy-argument-readonly.f90
index 97ae5bea85227b..6e0b21b41be688 100644
--- a/flang/test/Lower/dummy-argument-readonly.f90
+++ b/flang/test/Lower/dummy-argument-readonly.f90
@@ -8,51 +8,51 @@ subroutine scalar_intent_in(x)
integer, intent(in) :: x
end subroutine
! CHECK-LABEL: func.func @_QPscalar_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.read_only}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
subroutine real_scalar_intent_in(x)
real, intent(in) :: x
end subroutine
! CHECK-LABEL: func.func @_QPreal_scalar_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.ref<f32> {fir.bindc_name = "x", fir.read_only}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<f32> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
subroutine complex_scalar_intent_in(x)
complex, intent(in) :: x
end subroutine
! CHECK-LABEL: func.func @_QPcomplex_scalar_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.ref<complex<f32>> {fir.bindc_name = "x", fir.read_only}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<complex<f32>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
subroutine logical_scalar_intent_in(x)
logical, intent(in) :: x
end subroutine
! CHECK-LABEL: func.func @_QPlogical_scalar_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.ref<!fir.logical<4>> {fir.bindc_name = "x", fir.read_only}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.logical<4>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
recursive subroutine recursive_scalar_intent_in(x)
integer, intent(in) :: x
if (x > 0) call recursive_scalar_intent_in(x - 1)
end subroutine
! CHECK-LABEL: func.func @_QPrecursive_scalar_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.read_only}) attributes
+! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) attributes
subroutine optional_scalar_intent_in(x)
integer, intent(in), optional :: x
end subroutine
! CHECK-LABEL: func.func @_QPoptional_scalar_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.optional, fir.read_only}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.optional, fir.read_only}) {
subroutine target_scalar_intent_in(x)
integer, intent(in), target :: x
end subroutine
! CHECK-LABEL: func.func @_QPtarget_scalar_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.read_only, fir.target}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only, fir.target}) {
subroutine bindc_definition_intent_in(x) bind(c)
use iso_c_binding, only : c_int
integer(c_int), intent(in) :: x
end subroutine
! CHECK-LABEL: func.func @bindc_definition_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.read_only}) attributes
+! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) attributes
! Arrays are excluded because a forwarded dummy without an INTENT contract may
! require compiler-generated copy-out.
@@ -60,38 +60,38 @@ subroutine explicit_shape_intent_in(x)
integer, intent(in) :: x(10)
end subroutine
! CHECK-LABEL: func.func @_QPexplicit_shape_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<10xi32>> {fir.bindc_name = "x"}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<10xi32>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
subroutine assumed_size_intent_in(x)
integer, intent(in) :: x(*)
end subroutine
! CHECK-LABEL: func.func @_QPassumed_size_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<?xi32>> {fir.bindc_name = "x"}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<?xi32>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
subroutine target_explicit_shape_intent_in(x)
integer, intent(in), target :: x(10)
end subroutine
! CHECK-LABEL: func.func @_QPtarget_explicit_shape_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<10xi32>> {fir.bindc_name = "x", fir.target}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.array<10xi32>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.target}) {
subroutine assumed_shape_intent_in(x)
integer, intent(in) :: x(:)
end subroutine
! CHECK-LABEL: func.func @_QPassumed_shape_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "x"}) {
+! CHECK-SAME: %{{.*}}: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
subroutine assumed_rank_intent_in(x)
integer, intent(in) :: x(..)
end subroutine
! CHECK-LABEL: func.func @_QPassumed_rank_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.box<!fir.array<*:i32>> {fir.bindc_name = "x"}) {
+! CHECK-SAME: %{{.*}}: !fir.box<!fir.array<*:i32>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CHARACTER and derived types are outside the current conservative subset.
subroutine character_intent_in(x)
character(len=*), intent(in) :: x
end subroutine
! CHECK-LABEL: func.func @_QPcharacter_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.boxchar<1> {fir.bindc_name = "x"}) {
+! CHECK-SAME: %{{.*}}: !fir.boxchar<1> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
module readonly_derived_types
type :: type_with_array
@@ -102,20 +102,20 @@ subroutine derived_with_array_intent_in(x)
type(type_with_array), intent(in) :: x
end subroutine
! CHECK-LABEL: func.func @_QMreadonly_derived_typesPderived_with_array_intent_in(
-! CHECK-SAME: %{{.*}}: !fir.ref<!fir.type<{{.*}}>> {fir.bindc_name = "x"}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.type<{{.*}}>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
end module
subroutine intent_inout(x)
integer, intent(inout) :: x
end subroutine
! CHECK-LABEL: func.func @_QPintent_inout(
-! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x"}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_inout>}) {
subroutine intent_out(x)
integer, intent(out) :: x
end subroutine
! CHECK-LABEL: func.func @_QPintent_out(
-! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x"}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
subroutine intent_unspecified(x)
integer :: x
@@ -127,31 +127,31 @@ subroutine intent_in_value(x)
integer, intent(in), value :: x
end subroutine
! CHECK-LABEL: func.func @_QPintent_in_value(
-! CHECK-SAME: %{{.*}}: i32 {fir.bindc_name = "x"}) {
+! CHECK-SAME: %{{.*}}: i32 {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
subroutine intent_in_pointer(x)
integer, intent(in), pointer :: x
end subroutine
! CHECK-LABEL: func.func @_QPintent_in_pointer(
-! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "x", fir.read_only}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
subroutine intent_in_allocatable(x)
integer, intent(in), allocatable :: x
end subroutine
! CHECK-LABEL: func.func @_QPintent_in_allocatable(
-! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.heap<i32>>> {fir.bindc_name = "x", fir.read_only}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.heap<i32>>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
subroutine intent_in_pointer_array(x)
integer, intent(in), pointer :: x(:)
end subroutine
! CHECK-LABEL: func.func @_QPintent_in_pointer_array(
-! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "x", fir.read_only}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
subroutine intent_in_allocatable_array(x)
integer, intent(in), allocatable :: x(:)
end subroutine
! CHECK-LABEL: func.func @_QPintent_in_allocatable_array(
-! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "x", fir.read_only}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
! The marker is shallow for descriptor arguments. Defining a POINTER target is
! valid even though the descriptor itself has INTENT(IN).
@@ -160,14 +160,14 @@ subroutine intent_in_pointer_target_write(x)
x = 42
end subroutine
! CHECK-LABEL: func.func @_QPintent_in_pointer_target_write(
-! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "x", fir.read_only}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
! CHECK: hlfir.assign {{.*}} to {{.*}} : i32, !fir.ptr<i32>
subroutine intent_in_asynchronous(x)
integer, intent(in), asynchronous :: x
end subroutine
! CHECK-LABEL: func.func @_QPintent_in_asynchronous(
-! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.asynchronous, fir.bindc_name = "x"}) {
+! CHECK-SAME: %{{.*}}: !fir.ref<i32> {fir.asynchronous, fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! VOLATILE with INTENT(IN) is prohibited by C870 (Fortran 2023) and is already
! covered by test/Semantics/misc-declarations.f90, so it cannot be exercised by
diff --git a/flang/test/Lower/intentout-deallocate.f90 b/flang/test/Lower/intentout-deallocate.f90
index 8cfb213a5a0c5c..9c82b15211e7ff 100644
--- a/flang/test/Lower/intentout-deallocate.f90
+++ b/flang/test/Lower/intentout-deallocate.f90
@@ -48,7 +48,7 @@ subroutine sub1(a)
! Check inline deallocation of allocatable intent(out) on the callee side.
! CHECK-LABEL: func.func @_QMmod1Psub1(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a"})
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_out>})
! CHECK: %[[ARG0:.*]]:2 = hlfir.declare {{.*}}"_QMmod1Fsub1Ea"
! CHECK: %[[BOX:.*]] = fir.load %[[ARG0]]{{[#0]*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
@@ -114,7 +114,7 @@ subroutine sub5(t)
! is done with a runtime call.
! CHECK-LABEL: func.func @_QMmod1Psub5(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.type<_QMmod1Tt1{{[<]?}}{i:i32}{{[>]?}}>>>> {fir.bindc_name = "t"})
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.type<_QMmod1Tt1{{[<]?}}{i:i32}{{[>]?}}>>>> {fir.bindc_name = "t", fir.fortran_attrs = #fir.var_attrs<intent_out>})
! CHECK: %[[ARG0:.*]]:2 = hlfir.declare {{.*}}"_QMmod1Fsub5Et"
! CHECK: %[[BOX:.*]] = fir.load %[[ARG0]]{{[#0]*}} : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMmod1Tt1{{[<]?}}{i:i32}{{[>]?}}>>>>
! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.type<_QMmod1Tt1{{[<]?}}{i:i32}{{[>]?}}>>>) -> !fir.heap<!fir.type<_QMmod1Tt1{{[<]?}}{i:i32}{{[>]?}}>>
@@ -160,7 +160,7 @@ subroutine sub9(a)
! Check inline deallocation of optional allocatable intent(out) on the callee side.
! CHECK-LABEL: func.func @_QMmod1Psub9(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a", fir.optional})
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_out>, fir.optional})
! CHECK: %[[ARG0:.*]]:2 = hlfir.declare {{.*}}"_QMmod1Fsub9Ea"
! CHECK: %[[IS_PRESENT:.*]] = fir.is_present %[[ARG0]]{{[#0]*}} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> i1
! CHECK: fir.if %[[IS_PRESENT]] {
@@ -188,7 +188,7 @@ subroutine sub10(a)
end subroutine
! CHECK-LABEL: func.func @_QMmod1Psub10(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a"}) {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
! CHECK: %[[ARG0:.*]]:2 = hlfir.declare {{.*}}"_QMmod1Fsub10Ea"
! CHECK: %[[LOAD:.*]] = fir.load %[[ARG0]]{{[#0]*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
@@ -215,7 +215,7 @@ subroutine sub12(a)
end subroutine
! CHECK-LABEL: func.func @_QMmod1Psub12(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a"}) {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
! CHECK: %[[ARG0:.*]]:2 = hlfir.declare {{.*}}"_QMmod1Fsub12Ea"
! CHECK: %[[LOAD:.*]] = fir.load %[[ARG0]]{{[#0]*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
@@ -234,7 +234,7 @@ subroutine sub12(a)
! CHECK: }
! CHECK-LABEL: func.func @_QMmod1Psub13(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a"}) {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
! CHECK: %[[ARG0:.*]]:2 = hlfir.declare {{.*}}"_QMmod1Fsub12Ea"
! CHECK: %[[LOAD:.*]] = fir.load %[[ARG0]]{{[#0]*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
@@ -258,7 +258,7 @@ subroutine sub14(p)
end subroutine
! CHECK-LABEL: func.func @_QMmod1Psub14(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.type<_QMmod1Tt{a:i32}>>>> {fir.bindc_name = "p"}) {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.type<_QMmod1Tt{a:i32}>>>> {fir.bindc_name = "p", fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
! CHECK: %[[ARG0:.*]]:2 = hlfir.declare {{.*}}"_QMmod1Fsub14Ep"
! CHECK: %[[BOX:.*]] = fir.load %[[ARG0]]{{[#0]*}} : !fir.ref<!fir.class<!fir.heap<!fir.type<_QMmod1Tt{a:i32}>>>>
! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.class<!fir.heap<!fir.type<_QMmod1Tt{a:i32}>>>) -> !fir.heap<!fir.type<_QMmod1Tt{a:i32}>>
@@ -277,7 +277,7 @@ subroutine sub15(p)
end subroutine
! CHECK-LABEL: func.func @_QMmod1Psub15(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.heap<none>>> {fir.bindc_name = "p"}) {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.heap<none>>> {fir.bindc_name = "p", fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
! CHECK: %[[ARG0:.*]]:2 = hlfir.declare {{.*}}"_QMmod1Fsub15Ep"
! CHECK: %[[BOX:.*]] = fir.load %[[ARG0]]{{[#0]*}} : !fir.ref<!fir.class<!fir.heap<none>>>
! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.class<!fir.heap<none>>) -> !fir.heap<none>
@@ -295,7 +295,7 @@ subroutine sub16(p)
end subroutine
! CHECK-LABEL: func.func @_QMmod1Psub16(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.type<_QMmod1Tt{a:i32}>>>> {fir.bindc_name = "p", fir.optional}) {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.type<_QMmod1Tt{a:i32}>>>> {fir.bindc_name = "p", fir.fortran_attrs = #fir.var_attrs<intent_out>, fir.optional}) {
! CHECK: %[[ARG0:.*]]:2 = hlfir.declare {{.*}}"_QMmod1Fsub16Ep"
! CHECK: %[[IS_PRESENT:.*]] = fir.is_present %[[ARG0]]{{[#0]*}} : (!fir.ref<!fir.class<!fir.heap<!fir.type<_QMmod1Tt{a:i32}>>>>) -> i1
! CHECK: fir.if %[[IS_PRESENT]] {
diff --git a/flang/test/Lower/parent-component.f90 b/flang/test/Lower/parent-component.f90
index c8b464880e5f26..8c5c0593230adf 100644
--- a/flang/test/Lower/parent-component.f90
+++ b/flang/test/Lower/parent-component.f90
@@ -29,13 +29,13 @@ subroutine print_scalar(a)
type(p), intent(in) :: a
print*, a
end subroutine
- ! CHECK-LABEL: func.func private @_QFPprint_scalar(%{{.*}}: !fir.ref<!fir.type<_QFTp{a:i32}>> {fir.bindc_name = "a"})
+ ! CHECK-LABEL: func.func private @_QFPprint_scalar(%{{.*}}: !fir.ref<!fir.type<_QFTp{a:i32}>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>})
subroutine print_p(a)
type(p), intent(in) :: a(2)
print*, a
end subroutine
- ! CHECK-LABEL: func.func private @_QFPprint_p(%{{.*}}: !fir.ref<!fir.array<2x!fir.type<_QFTp{a:i32}>>> {fir.bindc_name = "a"})
+ ! CHECK-LABEL: func.func private @_QFPprint_p(%{{.*}}: !fir.ref<!fir.array<2x!fir.type<_QFTp{a:i32}>>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>})
subroutine init_with_slice()
type(c) :: y(2) = [ c(11, 21), c(12, 22) ]
diff --git a/flang/test/Lower/polymorphic-temp.f90 b/flang/test/Lower/polymorphic-temp.f90
index 956dfaf1b4dccf..8100f4c2732923 100644
--- a/flang/test/Lower/polymorphic-temp.f90
+++ b/flang/test/Lower/polymorphic-temp.f90
@@ -63,7 +63,7 @@ subroutine test_temp_from_intrinsic_reshape(i)
end subroutine
! CHECK-LABEL: func.func @_QMpoly_tmpPtest_temp_from_intrinsic_reshape(
-! CHECK-SAME: %[[I:.*]]: !fir.class<!fir.array<20x20xnone>> {fir.bindc_name = "i"}) {
+! CHECK-SAME: %[[I:.*]]: !fir.class<!fir.array<20x20xnone>> {fir.bindc_name = "i", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CHECK: %[[A:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?xnone>>> <{bindc_name = "a", uniq_name = "_QMpoly_tmpFtest_temp_from_intrinsic_reshapeEa"}>
! CHECK: %[[A_DECL:.*]]:2 = hlfir.declare %[[A]]
! CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I]]
@@ -85,7 +85,7 @@ subroutine test_temp_from_intrinsic_pack(i, mask)
end subroutine
! CHECK-LABEL: func.func @_QMpoly_tmpPtest_temp_from_intrinsic_pack(
-! CHECK-SAME: %[[I:.*]]: !fir.class<!fir.array<20x20x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "i"}, %[[MASK:.*]]: !fir.ref<!fir.array<20x20x!fir.logical<4>>> {fir.bindc_name = "mask"}) {
+! CHECK-SAME: %[[I:.*]]: !fir.class<!fir.array<20x20x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "i", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[MASK:.*]]: !fir.ref<!fir.array<20x20x!fir.logical<4>>> {fir.bindc_name = "mask", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CHECK: %[[TMP_RES:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpoly_tmpTp1{a:i32}>>>>
! CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I]]
! CHECK: %[[MASK_DECL:.*]]:2 = hlfir.declare %[[MASK]]
@@ -109,7 +109,7 @@ subroutine test_temp_from_unpack(v, m, f)
end subroutine
! CHECK-LABEL: func.func @_QMpoly_tmpPtest_temp_from_unpack(
-! CHECK-SAME: %[[V:.*]]: !fir.class<!fir.array<?x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "v"}, %[[M:.*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>> {fir.bindc_name = "m"}, %[[F:.*]]: !fir.class<!fir.array<?x?x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "f"}) {
+! CHECK-SAME: %[[V:.*]]: !fir.class<!fir.array<?x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "v", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[M:.*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>> {fir.bindc_name = "m", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[F:.*]]: !fir.class<!fir.array<?x?x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "f", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CHECK: %[[TMP_RES:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMpoly_tmpTp1{a:i32}>>>>
! CHECK: %[[F_DECL:.*]]:2 = hlfir.declare %[[F]]
! CHECK: %[[M_DECL:.*]]:2 = hlfir.declare %[[M]]
@@ -132,7 +132,7 @@ subroutine test_temp_from_intrinsic_cshift(a, shift)
end subroutine
! CHECK-LABEL: func.func @_QMpoly_tmpPtest_temp_from_intrinsic_cshift(
-! CHECK-SAME: %[[ARRAY:.*]]: !fir.class<!fir.array<20x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "a"}, %[[SHIFT:.*]]: !fir.ref<i32> {fir.bindc_name = "shift"}) {
+! CHECK-SAME: %[[ARRAY:.*]]: !fir.class<!fir.array<20x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[SHIFT:.*]]: !fir.ref<i32> {fir.bindc_name = "shift"}) {
! CHECK: %[[ARRAY_DECL:.*]]:2 = hlfir.declare %[[ARRAY]]
! CHECK: %[[SHIFT_DECL:.*]]:2 = hlfir.declare %[[SHIFT]]
! CHECK: %[[CSHIFT:.*]] = hlfir.cshift %[[ARRAY_DECL]]#0 %[[SHIFT_DECL]]#0 : (!fir.class<!fir.array<20x!fir.type<_QMpoly_tmpTp1{a:i32}>>>, !fir.ref<i32>) -> !hlfir.expr<20x!fir.type<_QMpoly_tmpTp1{a:i32}>?>
@@ -146,7 +146,7 @@ subroutine test_temp_from_intrinsic_eoshift(a, shift, b)
end subroutine
! CHECK-LABEL: func.func @_QMpoly_tmpPtest_temp_from_intrinsic_eoshift(
-! CHECK-SAME: %[[ARRAY:.*]]: !fir.class<!fir.array<20x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "a"}, %[[SHIFT:.*]]: !fir.ref<i32> {fir.bindc_name = "shift"}, %[[BOUNDARY:.*]]: !fir.class<!fir.type<_QMpoly_tmpTp1{a:i32}>> {fir.bindc_name = "b"}) {
+! CHECK-SAME: %[[ARRAY:.*]]: !fir.class<!fir.array<20x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[SHIFT:.*]]: !fir.ref<i32> {fir.bindc_name = "shift"}, %[[BOUNDARY:.*]]: !fir.class<!fir.type<_QMpoly_tmpTp1{a:i32}>> {fir.bindc_name = "b", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CHECK: %[[ARRAY_DECL:.*]]:2 = hlfir.declare %[[ARRAY]]
! CHECK: %[[BOUNDARY_DECL:.*]]:2 = hlfir.declare %[[BOUNDARY]]
! CHECK: %[[SHIFT_DECL:.*]]:2 = hlfir.declare %[[SHIFT]]
@@ -159,7 +159,7 @@ subroutine test_temp_from_intrinsic_transfer(source, mold)
end subroutine
! CHECK-LABEL: func.func @_QMpoly_tmpPtest_temp_from_intrinsic_transfer(
-! CHECK-SAME: %[[SOURCE:.*]]: !fir.class<!fir.array<?x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "source"}, %[[MOLD:.*]]: !fir.class<!fir.array<?x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "mold"}) {
+! CHECK-SAME: %[[SOURCE:.*]]: !fir.class<!fir.array<?x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "source", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[MOLD:.*]]: !fir.class<!fir.array<?x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "mold", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CHECK: %[[TMP_RES:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpoly_tmpTp1{a:i32}>>>>
! CHECK: %[[MOLD_DECL:.*]]:2 = hlfir.declare %[[MOLD]]
! CHECK: %[[SOURCE_DECL:.*]]:2 = hlfir.declare %[[SOURCE]]
@@ -174,7 +174,7 @@ subroutine test_temp_from_intrinsic_transpose(matrix)
end subroutine
! CHECK-LABEL: func.func @_QMpoly_tmpPtest_temp_from_intrinsic_transpose(
-! CHECK-SAME: %[[MATRIX:.*]]: !fir.class<!fir.array<?x?x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "matrix"}) {
+! CHECK-SAME: %[[MATRIX:.*]]: !fir.class<!fir.array<?x?x!fir.type<_QMpoly_tmpTp1{a:i32}>>> {fir.bindc_name = "matrix", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CHECK: %[[MATRIX_DECL:.*]]:2 = hlfir.declare %[[MATRIX]]
! CHECK: %[[TRANS:.*]] = hlfir.transpose %[[MATRIX_DECL]]#0 : (!fir.class<!fir.array<?x?x!fir.type<_QMpoly_tmpTp1{a:i32}>>>) -> !hlfir.expr<?x?x!fir.type<_QMpoly_tmpTp1{a:i32}>?>
@@ -189,7 +189,7 @@ subroutine test_merge_intrinsic(a, b)
end subroutine
! CHECK-LABEL: func.func @_QMpoly_tmpPtest_merge_intrinsic(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpoly_tmpTp1{a:i32}>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.class<!fir.type<_QMpoly_tmpTp1{a:i32}>> {fir.bindc_name = "b"}) {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpoly_tmpTp1{a:i32}>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[ARG1:.*]]: !fir.class<!fir.type<_QMpoly_tmpTp1{a:i32}>> {fir.bindc_name = "b", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CHECK: %[[A_DECL:.*]]:2 = hlfir.declare %[[ARG0]]
! CHECK: %[[B_DECL:.*]]:2 = hlfir.declare %[[ARG1]]
! CHECK: %[[DES_A:.*]] = hlfir.designate %[[A_DECL]]#0{"a"} : (!fir.class<!fir.type<_QMpoly_tmpTp1{a:i32}>>) -> !fir.ref<i32>
@@ -208,7 +208,7 @@ subroutine test_merge_intrinsic2(a, b, i)
end subroutine
! CHECK-LABEL: func.func @_QMpoly_tmpPtest_merge_intrinsic2(
-! CHECK-SAME: %[[A:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.type<_QMpoly_tmpTp1{a:i32}>>>> {fir.bindc_name = "a", fir.read_only}, %[[B:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.type<_QMpoly_tmpTp1{a:i32}>>>> {fir.bindc_name = "b"}, %[[I:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.read_only}) {
+! CHECK-SAME: %[[A:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.type<_QMpoly_tmpTp1{a:i32}>>>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}, %[[B:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.type<_QMpoly_tmpTp1{a:i32}>>>> {fir.bindc_name = "b"}, %[[I:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
! CHECK: %[[A_DECL:.*]]:2 = hlfir.declare %[[A]]
! CHECK: %[[B_DECL:.*]]:2 = hlfir.declare %[[B]]
! CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I]]
@@ -230,7 +230,7 @@ subroutine test_merge_intrinsic3(a, b, i)
end subroutine
! CHECK-LABEL: func.func @_QMpoly_tmpPtest_merge_intrinsic3(
-! CHECK-SAME: %[[A:.*]]: !fir.class<none> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.class<none> {fir.bindc_name = "b"}, %[[I:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.read_only}) {
+! CHECK-SAME: %[[A:.*]]: !fir.class<none> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[B:.*]]: !fir.class<none> {fir.bindc_name = "b", fir.fortran_attrs = #fir.var_attrs<intent_in>}, %[[I:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
! CHECK: %[[A_DECL:.*]]:2 = hlfir.declare %[[A]]
! CHECK: %[[B_DECL:.*]]:2 = hlfir.declare %[[B]]
! CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I]]
@@ -247,7 +247,7 @@ subroutine test_merge_intrinsic4(i)
end subroutine
! CHECK-LABEL: func.func @_QMpoly_tmpPtest_merge_intrinsic4(
-! CHECK-SAME: %[[I:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.read_only}) {
+! CHECK-SAME: %[[I:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
! CHECK: %[[V_0:[0-9]+]] = fir.alloca !fir.class<!fir.heap<none>> <{bindc_name = "a", uniq_name = "_QMpoly_tmpFtest_merge_intrinsic4Ea"}>
! CHECK: %[[V_1:[0-9]+]] = fir.zero_bits !fir.heap<none>
! CHECK: %[[V_2:[0-9]+]] = fir.embox %[[V_1]] : (!fir.heap<none>) -> !fir.class<!fir.heap<none>>
@@ -274,7 +274,7 @@ subroutine test_merge_intrinsic5(i)
end subroutine
! CHECK-LABEL: func.func @_QMpoly_tmpPtest_merge_intrinsic5(
-! CHECK-SAME: %[[I:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.read_only}) {
+! CHECK-SAME: %[[I:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
! CHECK: %[[V_0:[0-9]+]] = fir.alloca !fir.class<!fir.ptr<none>> <{bindc_name = "a", uniq_name = "_QMpoly_tmpFtest_merge_intrinsic5Ea"}>
! CHECK: %[[V_1:[0-9]+]] = fir.zero_bits !fir.ptr<none>
! CHECK: %[[V_2:[0-9]+]] = fir.embox %[[V_1]] : (!fir.ptr<none>) -> !fir.class<!fir.ptr<none>>
diff --git a/flang/test/Lower/polymorphic.f90 b/flang/test/Lower/polymorphic.f90
index 2cf2d517edcfa7..8d6f192935b643 100644
--- a/flang/test/Lower/polymorphic.f90
+++ b/flang/test/Lower/polymorphic.f90
@@ -66,7 +66,7 @@ elemental subroutine assign_p1_int(lhs, rhs)
lhs%b = rhs
End Subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPassign_p1_int(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "lhs"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "rhs", fir.read_only}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "lhs", fir.fortran_attrs = #fir.var_attrs<intent_inout>}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "rhs", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}
! CHECK-SAME: attributes {fir.proc_attrs = #fir.proc_attrs<elemental, pure>}
! CHECK: %[[LHS:.*]]:2 = hlfir.declare %[[ARG0]]{{.*}}uniq_name("_QMpolymorphic_testFassign_p1_intElhs") fortran_attrs<intent_inout>
! CHECK: %[[RHS:.*]]:2 = hlfir.declare %[[ARG1]]{{.*}}uniq_name("_QMpolymorphic_testFassign_p1_intErhs") fortran_attrs<intent_in>
@@ -80,7 +80,7 @@ elemental integer function elemental_fct(this)
elemental_fct = this%a
end function
! CHECK-LABEL: func.func @_QMpolymorphic_testPelemental_fct(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "this"})
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "this", fir.fortran_attrs = #fir.var_attrs<intent_in>})
! CHECK-SAME: -> i32 attributes {fir.proc_attrs = #fir.proc_attrs<elemental, pure>}
! CHECK: %[[THIS:.*]]:2 = hlfir.declare %[[ARG0]]{{.*}}uniq_name("_QMpolymorphic_testFelemental_fctEthis") fortran_attrs<intent_in>
! CHECK: %[[A:.*]] = hlfir.designate %[[THIS]]#0{"a"}
@@ -90,7 +90,7 @@ elemental subroutine elemental_sub(this)
this%a = this%a * this%b
end subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPelemental_sub(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "this"})
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "this", fir.fortran_attrs = #fir.var_attrs<intent_inout>})
! CHECK-SAME: attributes {fir.proc_attrs = #fir.proc_attrs<elemental, pure>}
! CHECK: %[[THIS:.*]]:2 = hlfir.declare %[[ARG0]]
! CHECK: hlfir.designate %[[THIS]]#0{"a"}
@@ -102,7 +102,7 @@ elemental subroutine elemental_sub_pass(c, this)
this%a = this%a * this%b + c
end subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPelemental_sub_pass(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "c", fir.read_only}, %[[ARG1:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "this"})
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "c", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}, %[[ARG1:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "this", fir.fortran_attrs = #fir.var_attrs<intent_inout>})
! CHECK-SAME: attributes {fir.proc_attrs = #fir.proc_attrs<elemental, pure>}
logical elemental function lt(i, poly)
@@ -111,7 +111,7 @@ logical elemental function lt(i, poly)
lt = i < poly%a
End Function
! CHECK-LABEL: func.func @_QMpolymorphic_testPlt(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.read_only}, %[[ARG1:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "poly"})
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}, %[[ARG1:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "poly", fir.fortran_attrs = #fir.var_attrs<intent_in>})
! CHECK-SAME: -> !fir.logical<4> attributes {fir.proc_attrs = #fir.proc_attrs<elemental, pure>}
! CHECK: %[[I:.*]]:2 = hlfir.declare %[[ARG0]]{{.*}}uniq_name("_QMpolymorphic_testFltEi") fortran_attrs<intent_in>
! CHECK: %[[POLY:.*]]:2 = hlfir.declare %[[ARG1]]{{.*}}uniq_name("_QMpolymorphic_testFltEpoly") fortran_attrs<intent_in>
@@ -310,7 +310,7 @@ subroutine takes_p1(p)
class(p1), intent(in) :: p
end subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPtakes_p1(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "p"}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "p", fir.fortran_attrs = #fir.var_attrs<intent_in>}
! TODO: implement polymorphic temporary in lowering
! subroutine no_reassoc_poly_value(a, i)
@@ -363,7 +363,7 @@ subroutine up_input(a)
class(*), intent(in) :: a
end subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPup_input(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a"}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>}
subroutine pass_trivial_to_up()
call up_input('hello')
@@ -389,7 +389,7 @@ subroutine up_arr_input(a)
class(*), intent(in) :: a(2)
end subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPup_arr_input(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.array<2xnone>> {fir.bindc_name = "a"}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.array<2xnone>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>}
subroutine pass_trivial_arr_to_up()
character :: c(2)
@@ -550,7 +550,7 @@ subroutine write_p1(dtv, unit, iotype, v_list, iostat, iomsg)
! dummy subroutine for testing purpose
end subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPwrite_p1(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "dtv"}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "dtv", fir.fortran_attrs = #fir.var_attrs<intent_in>}
subroutine read_p1(dtv, unit, iotype, v_list, iostat, iomsg)
class(p1), intent(inout) :: dtv
@@ -562,7 +562,7 @@ subroutine read_p1(dtv, unit, iotype, v_list, iostat, iomsg)
! dummy subroutine for testing purpose
end subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPread_p1(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "dtv"}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "dtv", fir.fortran_attrs = #fir.var_attrs<intent_inout>}
subroutine test_polymorphic_io()
type(p1), target :: t
@@ -600,13 +600,13 @@ subroutine test_unlimited_polymorphic_intentout(a)
class(*), intent(out) :: a
end subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPtest_unlimited_polymorphic_intentout(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a"}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_out>}
subroutine test_polymorphic_intentout(a)
class(p1), intent(out) :: a
end subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPtest_polymorphic_intentout(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a"}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_out>}
subroutine rebox_up_to_record_type(p)
class(*), allocatable, target :: p(:,:)
@@ -680,13 +680,13 @@ subroutine opt_int(i)
call opt_up(i)
end subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPopt_int(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.optional, fir.read_only}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "i", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.optional, fir.read_only}
subroutine opt_up(up)
class(*), optional, intent(in) :: up
end subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPopt_up(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "up", fir.optional}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "up", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.optional}
function rhs()
class(p1), pointer :: rhs
@@ -714,7 +714,7 @@ subroutine up_pointer(p)
class(*), pointer, intent(in) :: p
end subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPup_pointer(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.ptr<none>>> {fir.bindc_name = "p", fir.read_only}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.ptr<none>>> {fir.bindc_name = "p", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}
subroutine test_char_to_up_pointer(c)
character(*), target :: c
@@ -865,7 +865,7 @@ subroutine pass_up(up)
class(*), intent(in) :: up
end subroutine
! CHECK-LABEL: func.func @_QMpolymorphic_testPpass_up(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "up"}
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "up", fir.fortran_attrs = #fir.var_attrs<intent_in>}
! TODO: unlimited polymorphic temporary in lowering
! subroutine parenthesized_up(a)
diff --git a/flang/test/Lower/repack-arrays-finalized-dummy.f90 b/flang/test/Lower/repack-arrays-finalized-dummy.f90
index 0c0a914d080565..a9e132cad31f9f 100644
--- a/flang/test/Lower/repack-arrays-finalized-dummy.f90
+++ b/flang/test/Lower/repack-arrays-finalized-dummy.f90
@@ -17,7 +17,7 @@ end subroutine my_final
end interface
contains
! CHECK-LABEL: func.func @_QMmPtest(
-! CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.class<!fir.array<?x!fir.type<_QMmTt>>> {fir.bindc_name = "x"}) {
+! CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.class<!fir.array<?x!fir.type<_QMmTt>>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
subroutine test(x)
class(t), intent(out) :: x(:)
! CHECK: %[[VAL_2:.*]] = fir.pack_array %[[VAL_0]] heap whole : (!fir.class<!fir.array<?x!fir.type<_QMmTt>>>) -> !fir.class<!fir.array<?x!fir.type<_QMmTt>>>
diff --git a/flang/test/Lower/repack-arrays.f90 b/flang/test/Lower/repack-arrays.f90
index 339b4a723abc88..d7a7fbadf12921 100644
--- a/flang/test/Lower/repack-arrays.f90
+++ b/flang/test/Lower/repack-arrays.f90
@@ -64,7 +64,7 @@ subroutine test3(x)
end subroutine test3
! ALL-LABEL: func.func @_QPtest4(
-! ALL-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
+! ALL-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_inout>}) {
subroutine test4(x)
real, intent(inout) :: x(:)
! ALL: %[[VAL_2:.*]] = fir.pack_array %[[VAL_0]]
@@ -82,7 +82,7 @@ subroutine test4(x)
end subroutine test4
! ALL-LABEL: func.func @_QPtest5(
-! ALL-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
+! ALL-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
subroutine test5(x)
real, intent(in) :: x(:)
! ALL: %[[VAL_2:.*]] = fir.pack_array %[[VAL_0]]
@@ -99,7 +99,7 @@ subroutine test5(x)
end subroutine test5
! ALL-LABEL: func.func @_QPtest6(
-! ALL-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
+! ALL-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
subroutine test6(x)
real, intent(out) :: x(:)
! ALL: %[[VAL_2:.*]] = fir.pack_array %[[VAL_0]]
diff --git a/flang/test/Lower/select-type-2.f90 b/flang/test/Lower/select-type-2.f90
index 57c4661d0115e8..d32e9aa8cf44b1 100644
--- a/flang/test/Lower/select-type-2.f90
+++ b/flang/test/Lower/select-type-2.f90
@@ -29,7 +29,7 @@ subroutine select_type1(a)
end subroutine
! CHECK-LABEL: func.func @_QMselect_type_2Pselect_type1(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_2Tp1{a:i32,b:i32}>> {fir.bindc_name = "a"}) {
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_2Tp1{a:i32,b:i32}>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CHECK: %[[A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope {{.*}} uniq_name("_QMselect_type_2Fselect_type1Ea") fortran_attrs<intent_in>
! CHECK: %[[TDESC_P3_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_2Tp3
! CHECK: %[[TDESC_P3_CONV:.*]] = fir.convert %[[TDESC_P3_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
diff --git a/flang/test/Lower/select-type.f90 b/flang/test/Lower/select-type.f90
index 4fa31d9180eaca..58a0d4fae6db23 100644
--- a/flang/test/Lower/select-type.f90
+++ b/flang/test/Lower/select-type.f90
@@ -55,7 +55,7 @@ subroutine select_type1(a)
end subroutine
! CHECK-LABEL: func.func @_QMselect_type_lower_testPselect_type1(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a"})
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>})
! CHECK: %[[A:.*]]:2 = hlfir.declare %[[ARG0]]
! CHECK: fir.select_type %[[A]]#1 : !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>
! CHECK-SAME: [#fir.type_is<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>, ^[[TYPE_IS_BLK:.*]], #fir.class_is<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>, ^[[CLASS_IS_P1_BLK:.*]], #fir.class_is<!fir.type<_QMselect_type_lower_testTp2{{.*}}>>, ^[[CLASS_IS_P2_BLK:.*]], unit, ^[[DEFAULT_BLOCK:.*]]]
@@ -68,7 +68,7 @@ subroutine select_type1(a)
! CHECK: ^[[DEFAULT_BLOCK]]
! CFG-LABEL: func.func @_QMselect_type_lower_testPselect_type1(
-! CFG-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a"}) {
+! CFG-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CFG: %[[A:.*]]:2 = hlfir.declare %[[ARG0]]
! CFG: %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
! CFG: %[[BOX_TDESC:.*]] = fir.box_tdesc %[[A]]#1 : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>
@@ -166,7 +166,7 @@ subroutine select_type3(a)
end subroutine
! CHECK-LABEL: func.func @_QMselect_type_lower_testPselect_type3(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>>> {fir.bindc_name = "a", fir.read_only})
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only})
! CHECK: %[[A:.*]]:2 = hlfir.declare %[[ARG0]]{{.*}}uniq_name("_QMselect_type_lower_testFselect_type3Ea") fortran_attrs<intent_in, pointer>
! CHECK: %[[ARG0_LOAD:.*]] = fir.load %[[A]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>>>
! CHECK: %[[SELECTOR:.*]] = hlfir.designate %[[ARG0_LOAD]] (%{{.*}}) : (!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>>, index) -> !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>
@@ -177,7 +177,7 @@ subroutine select_type3(a)
! CHECK: ^[[DEFAULT_BLK]]
! CFG-LABEL: func.func @_QMselect_type_lower_testPselect_type3(
-! CFG-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>>> {fir.bindc_name = "a", fir.read_only}) {
+! CFG-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) {
! CFG: %[[SELECTOR:.*]] = hlfir.designate %{{.*}} (%{{.*}}) : (!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>>, index) -> !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>
! CFG: %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
! CFG: %[[SELECTOR_TDESC:.*]] = fir.box_tdesc %[[SELECTOR]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<{{.*}}>
@@ -215,7 +215,7 @@ subroutine select_type4(a)
end subroutine
! CHECK-LABEL: func.func @_QMselect_type_lower_testPselect_type4(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a"})
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>})
! CHECK: %[[A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope {{.*}} uniq_name("_QMselect_type_lower_testFselect_type4Ea") fortran_attrs<intent_in>
! CHECK: fir.select_type %[[A]]#1 : !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>
! CHECK-SAME: [#fir.type_is<!fir.type<_QMselect_type_lower_testTp3K8{{.*}}>>, ^[[P3_8:.*]], #fir.type_is<!fir.type<_QMselect_type_lower_testTp3K4{{.*}}>>, ^[[P3_4:.*]], #fir.class_is<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>, ^[[P1:.*]], unit, ^[[EXIT:.*]]]
@@ -225,7 +225,7 @@ subroutine select_type4(a)
! CHECK: ^[[EXIT]]
! CFG-LABEL: func.func @_QMselect_type_lower_testPselect_type4(
-! CFG-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a"}) {
+! CFG-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CFG: %[[A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope {{.*}} uniq_name("_QMselect_type_lower_testFselect_type4Ea") fortran_attrs<intent_in>
! CFG: %[[TDESC_P3_8_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp3K8
! CFG: %[[BOX_TDESC:.*]] = fir.box_tdesc %[[A]]#1 : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<{{.*}}>
@@ -278,7 +278,7 @@ subroutine select_type5(a)
end subroutine
! CHECK-LABEL: func.func @_QMselect_type_lower_testPselect_type5(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a"})
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>})
! CHECK: %[[A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} arg 1 uniq_name("_QMselect_type_lower_testFselect_type5Ea") fortran_attrs<intent_in> : (!fir.class<none>, !fir.dscope) -> (!fir.class<none>, !fir.class<none>)
! CHECK: fir.select_type %[[A]]#1 : !fir.class<none>
! CHECK-SAME: [#fir.type_is<i8>, ^[[I8_BLK:.*]], #fir.type_is<i32>, ^[[I32_BLK:.*]], #fir.type_is<f32>, ^[[F32_BLK:.*]], #fir.type_is<!fir.logical<4>>, ^[[LOG_BLK:.*]], #fir.type_is<!fir.char<1,?>>, ^[[CHAR_BLK:.*]], unit, ^[[DEFAULT:.*]]]
@@ -290,7 +290,7 @@ subroutine select_type5(a)
! CHECK: ^[[DEFAULT]]
! CFG-LABEL: func.func @_QMselect_type_lower_testPselect_type5(
-! CFG-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a"}) {
+! CFG-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_in>}) {
! CFG: %[[A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} arg 1 uniq_name("_QMselect_type_lower_testFselect_type5Ea") fortran_attrs<intent_in> : (!fir.class<none>, !fir.dscope) -> (!fir.class<none>, !fir.class<none>)
! CFG: %[[INT8_TC:.*]] = arith.constant 7 : i8
! CFG: %[[TYPE_CODE:.*]] = fir.box_typecode %[[A]]#1 : (!fir.class<none>) -> i8
@@ -411,7 +411,7 @@ subroutine select_type7(a)
end subroutine
! CHECK-LABEL: func.func @_QMselect_type_lower_testPselect_type7(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a"})
+! CHECK-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_out>})
! CHECK: %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[DSCOPE]] arg 1 {{.*}}uniq_name("_QMselect_type_lower_testFselect_type7Ea"){{.*}} : (!fir.class<none>, !fir.dscope) -> (!fir.class<none>, !fir.class<none>)
! CHECK: fir.select_type %[[DECL]]#1 :
@@ -421,7 +421,7 @@ subroutine select_type7(a)
! class is (p4) -> class is (p2) -> class is (p1) -> class default
! CFG-LABEL: func.func @_QMselect_type_lower_testPselect_type7(
-! CFG-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a"}) {
+! CFG-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a", fir.fortran_attrs = #fir.var_attrs<intent_out>}) {
! CFG: %[[TDESC_P4_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp4
! CFG: %[[TDESC_P4_CONV:.*]] = fir.convert %[[TDESC_P4_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
! CFG: %[[BOX_NONE:.*]] = fir.convert %{{.*}} : (!fir.class<none>) -> !fir.box<none>
diff --git a/flang/test/Lower/unsigned-ops.f90 b/flang/test/Lower/unsigned-ops.f90
index c6b0521c1e2947..820cca63ec330e 100644
--- a/flang/test/Lower/unsigned-ops.f90
+++ b/flang/test/Lower/unsigned-ops.f90
@@ -5,7 +5,7 @@ unsigned function f01(u, v)
f01 = u + v - 1u
end
-!CHECK: func.func @_QPf01(%[[ARG0:.*]]: !fir.ref<ui32> {fir.bindc_name = "u", fir.read_only}, %[[ARG1:.*]]: !fir.ref<ui32> {fir.bindc_name = "v", fir.read_only}) -> ui32 {
+!CHECK: func.func @_QPf01(%[[ARG0:.*]]: !fir.ref<ui32> {fir.bindc_name = "u", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}, %[[ARG1:.*]]: !fir.ref<ui32> {fir.bindc_name = "v", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) -> ui32 {
!CHECK: %[[C1_I32:.*]] = arith.constant 1 : i32
!CHECK: %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
!CHECK: %[[VAL_1:.*]] = fir.alloca ui32 <{bindc_name = "f01", uniq_name = "_QFf01Ef01"}>
@@ -30,7 +30,7 @@ unsigned function f02(u, v)
f02 = u ** v - 1u
end
-!CHECK: func.func @_QPf02(%[[ARG0:.*]]: !fir.ref<ui32> {fir.bindc_name = "u", fir.read_only}, %[[ARG1:.*]]: !fir.ref<ui32> {fir.bindc_name = "v", fir.read_only}) -> ui32 {
+!CHECK: func.func @_QPf02(%[[ARG0:.*]]: !fir.ref<ui32> {fir.bindc_name = "u", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}, %[[ARG1:.*]]: !fir.ref<ui32> {fir.bindc_name = "v", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) -> ui32 {
!CHECK: %[[C1_i32:.*]] = arith.constant 1 : i32
!CHECK: %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
!CHECK: %[[VAL_1:.*]] = fir.alloca ui32 <{bindc_name = "f02", uniq_name = "_QFf02Ef02"}>
diff --git a/flang/test/Lower/volatile-string.f90 b/flang/test/Lower/volatile-string.f90
index 11fdf95f00368b..65db6c89b19792 100644
--- a/flang/test/Lower/volatile-string.f90
+++ b/flang/test/Lower/volatile-string.f90
@@ -69,7 +69,7 @@ subroutine assign_different_length(string)
! CHECK: }
! CHECK-LABEL: func.func private @_QFPassign_same_length(
-! CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.boxchar<1> {fir.bindc_name = "x"}) attributes {fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage<internal>} {
+! CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.boxchar<1> {fir.bindc_name = "x", fir.fortran_attrs = #fir.var_attrs<intent_inout>}) attributes {fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage<internal>} {
! CHECK: %[[VAL_1:.*]] = arith.constant 3 : index
! CHECK: %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
! CHECK: %[[VAL_3:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
@@ -83,7 +83,7 @@ subroutine assign_different_length(string)
! CHECK: }
! CHECK-LABEL: func.func private @_QFPassign_different_length(
-! CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.boxchar<1> {fir.bindc_name = "string"}) attributes {fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage<internal>} {
+! CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.boxchar<1> {fir.bindc_name = "string", fir.fortran_attrs = #fir.var_attrs<intent_inout>}) attributes {fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage<internal>} {
! CHECK: %[[VAL_1:.*]] = arith.constant 2 : index
! CHECK: %[[VAL_2:.*]] = arith.constant 3 : index
! CHECK: %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
diff --git a/flang/test/Lower/volatile1.f90 b/flang/test/Lower/volatile1.f90
index 98e88662e59aa5..e7f91edb255b19 100644
--- a/flang/test/Lower/volatile1.f90
+++ b/flang/test/Lower/volatile1.f90
@@ -66,7 +66,7 @@ subroutine declared_volatile_in_this_scope(v,n)
! CHECK: }
! CHECK-LABEL: func.func private @_QFPnot_declared_volatile_in_this_scope(
-! CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<i32> {fir.bindc_name = "v"}) attributes {{.+}} {
+! CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<i32> {fir.bindc_name = "v", fir.fortran_attrs = #fir.var_attrs<intent_inout>}) attributes {{.+}} {
! CHECK: %[[VAL_1:.*]] = arith.constant 1 : i32
! CHECK: %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {{.+}} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -75,8 +75,8 @@ subroutine declared_volatile_in_this_scope(v,n)
! CHECK: }
! CHECK-LABEL: func.func private @_QFPdeclared_volatile_in_this_scope(
-! CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<!fir.array<?xi32>> {fir.bindc_name = "v"},
-! CHECK-SAME: %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<i32> {fir.bindc_name = "n", fir.read_only}) attributes {{.+}} {
+! CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<!fir.array<?xi32>> {fir.bindc_name = "v", fir.fortran_attrs = #fir.var_attrs<intent_inout>},
+! CHECK-SAME: %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<i32> {fir.bindc_name = "n", fir.fortran_attrs = #fir.var_attrs<intent_in>, fir.read_only}) attributes {{.+}} {
! CHECK: %[[VAL_2:.*]] = arith.constant 1 : i32
! CHECK: %[[VAL_3:.*]] = arith.constant 0 : index
! CHECK: %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
>From fd2e4b3941ef6f01a26a178f86639cd2ecf43243 Mon Sep 17 00:00:00 2001
From: Grigory Pastukhov <gpastukhov at meta.com>
Date: Tue, 6 Oct 2026 14:42:56 -0700
Subject: [PATCH 32/46] [X86] Mark .lrodata for non-mergeable constants with
SHF_X86_64_LARGE (#229279)
Under the large code model, a constant pool entry that is not mergeable
(a size other than 4, 8, 16 or 32 bytes, such as a 64-byte AVX-512
vector) is placed in .lrodata without SHF_X86_64_LARGE. Sections are
uniqued by name, so a large global emitted into .lrodata later in the
same module ends up in that unflagged section.
Assisted-By: Claude Opus 5.5
---
llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp | 5 +++--
.../test/CodeGen/X86/code-model-elf-constpool-large.ll | 10 ++++++++++
llvm/test/CodeGen/X86/morestack-decl.ll | 2 +-
3 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 16912083af9f9f..002e9bf6321c65 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -1109,8 +1109,9 @@ MCSection *TargetLoweringObjectFileELF::getSectionForConstantImpl(
return Context.getELFSection(CstPrefix + ".cst32" + SectionSuffixStr,
ELF::SHT_PROGBITS, MergeableCstFlags, 32);
if (Kind.isReadOnly())
- return Context.getELFSection(CstPrefix + SectionSuffixStr,
- ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
+ return Context.getELFSection(
+ CstPrefix + SectionSuffixStr, ELF::SHT_PROGBITS,
+ ELF::SHF_ALLOC | (IsLarge ? ELF::SHF_X86_64_LARGE : 0));
assert(Kind.isReadOnlyWithRel() && "Unknown section kind");
return Context.getELFSection(".data.rel.ro" + SectionSuffixStr,
diff --git a/llvm/test/CodeGen/X86/code-model-elf-constpool-large.ll b/llvm/test/CodeGen/X86/code-model-elf-constpool-large.ll
index e2941783b14f92..b694b95e9d3a57 100644
--- a/llvm/test/CodeGen/X86/code-model-elf-constpool-large.ll
+++ b/llvm/test/CodeGen/X86/code-model-elf-constpool-large.ll
@@ -12,9 +12,11 @@
; CHECK: .lrodata.cst16 {{.*}} AMl
; CHECK: .lrodata.cst4 {{.*}} AMl
+; CHECK: .lrodata {{.*}} Al
; MED: .rodata.cst16 {{.*}} AM
; MED: .rodata.cst4 {{.*}} AM
+; MED: .rodata {{.*}} A {{.*}}
; ASM-MED: movaps .LCPI0_0(%rip),
; ASM-LARGE: movabsq $.LCPI0_0 at GOTOFF,
@@ -67,6 +69,14 @@ define float @scalar_const(float %x) {
ret float %r
}
+; A 64-byte vector constant is not mergeable and goes to .lrodata.
+define <16 x float> @wide() "target-features"="+avx512f" {
+ ret <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0,
+ float 5.0, float 6.0, float 7.0, float 8.0,
+ float 9.0, float 10.0, float 11.0, float 12.0,
+ float 13.0, float 14.0, float 15.0, float 16.0>
+}
+
!llvm.module.flags = !{!1}
!1 = !{i32 1, !"ProfileSummary", !2}
diff --git a/llvm/test/CodeGen/X86/morestack-decl.ll b/llvm/test/CodeGen/X86/morestack-decl.ll
index 8a1d739e5380e4..75048ba05e5116 100644
--- a/llvm/test/CodeGen/X86/morestack-decl.ll
+++ b/llvm/test/CodeGen/X86/morestack-decl.ll
@@ -3,7 +3,7 @@
; Check what happens if we have an existing declaration of __morestack_addr
; CHECK: .section ".note.GNU-stack","", at progbits
-; CHECK-NEXT: .section .lrodata,"a", at progbits
+; CHECK-NEXT: .section .lrodata,"al", at progbits
; CHECK-NEXT: __morestack_addr:
; CHECK-NEXT: .quad __morestack
>From 45cba0f73a84ec8d016b4060d9d9ec4892ab5acf Mon Sep 17 00:00:00 2001
From: Alexey Samsonov <vonosmas at gmail.com>
Date: Tue, 6 Oct 2026 14:44:20 -0700
Subject: [PATCH 33/46] [libc] Use sighandler_t typedef in public headers for
all platforms. (#229508)
`signal` function from ANSI C takes an argument and returns value of
type `typeof(void(int))`.
While not strictly necessary, some libc implementations provide their
own convenience typedef for the signal handler type. LLVM-libc used
`sighandler_t` previously, but only on Linux systems. This change makes
this typedef used on other systems as well, the reasons being:
* convenience for the user code, which can rely on this typedef in the
signal-handling code;
* ability to fix a gross hdrgen workaround (YAML configs and
hdrgen-based generation doesn't really support a C-style syntax for
functions-returning-function-pointers)
* ability to fix a **bug** in the generated header - `__NOEXCEPT` suffix
is translated into `noexcept` in the C++ mode, with this qualifier
applied to the _returned function pointer type_ instead of the `signal`
function itself. The "proper" way to use noexcept would be to move it
inside the parentheses (which hdrgen is not capable of today), but that
also wouldn't work because in C mode `__NOEXCEPT` is an attribute, which
can't be located at that spot.
To address all of the above, it's easier to just make `sighandler_t` an
LLVM-libc extension to C/POSIX (which GNU does as well).
---
libc/include/llvm-libc-types/sighandler_t.h | 3 ---
libc/include/signal.yaml | 10 +++-------
2 files changed, 3 insertions(+), 10 deletions(-)
diff --git a/libc/include/llvm-libc-types/sighandler_t.h b/libc/include/llvm-libc-types/sighandler_t.h
index f39ab04685200b..d42168093fa9ae 100644
--- a/libc/include/llvm-libc-types/sighandler_t.h
+++ b/libc/include/llvm-libc-types/sighandler_t.h
@@ -9,9 +9,6 @@
#ifndef LLVM_LIBC_TYPES_SIGHANDLER_T_H
#define LLVM_LIBC_TYPES_SIGHANDLER_T_H
-#ifdef __linux__
-// For compatibility with glibc.
typedef void (*sighandler_t)(int);
-#endif
#endif // LLVM_LIBC_TYPES_SIGHANDLER_T_H
diff --git a/libc/include/signal.yaml b/libc/include/signal.yaml
index f5f110915cfc8c..5c73f6dd7994eb 100644
--- a/libc/include/signal.yaml
+++ b/libc/include/signal.yaml
@@ -37,6 +37,7 @@ types:
- type_name: sighandler_t
standards:
- gnu
+ - llvm_libc_ext
- type_name: siginfo_t
- type_name: sigset_t
- type_name: stack_t
@@ -106,15 +107,10 @@ functions:
- name: signal
standards:
- stdc
- # May the Geneva Convention have mercy on my soul... Why this insanity?
- # Well: signal returns a function pointer to a function with no return
- # value and which accepts an int. The parameter list appears on the far
- # right of the declaration. i.e.
- # void (*signal(int, void (*)(int)))(int);
- return_type: void (*
+ return_type: sighandler_t
arguments:
- type: int
- - type: void (*)(int)))(int
+ - type: sighandler_t
- name: sigprocmask
standards:
- posix
>From 48884639e2dfdef3a2d14f6f32f1f17862cc3e48 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Tue, 6 Oct 2026 23:44:44 +0200
Subject: [PATCH 34/46] RuntimeLibcallsEmitter: Handle calling conv in
per-library functions (#223760)
Pull handling of the default calling convention into the
setAvailableLibFuncs_<name> functions, so the library logic will be
fully contained.
Co-authored-by: Claude (Opus 4.8) <noreply at anthropic.com>
---
...untimeLibcallEmitter-library-default-cc.td | 50 +++++++++++
.../TableGen/Basic/RuntimeLibcallsEmitter.cpp | 85 +++++++++++++++++--
2 files changed, 130 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/TableGen/RuntimeLibcallEmitter-library-default-cc.td
diff --git a/llvm/test/TableGen/RuntimeLibcallEmitter-library-default-cc.td b/llvm/test/TableGen/RuntimeLibcallEmitter-library-default-cc.td
new file mode 100644
index 00000000000000..5111d327f8a3e1
--- /dev/null
+++ b/llvm/test/TableGen/RuntimeLibcallEmitter-library-default-cc.td
@@ -0,0 +1,50 @@
+// RUN: llvm-tblgen -gen-runtime-libcalls -I %p/../../include %s | FileCheck %s
+
+// A member CC naming the DefaultCC sentinel gets a DefaultCC local
+// seeded from the consuming system library's
+// DefaultLibcallCallingConv.
+
+include "llvm/IR/RuntimeLibcallsImpl.td"
+
+def SOME_CALL : RuntimeLibcall;
+def OTHER_CALL : RuntimeLibcall;
+
+def some_impl : RuntimeLibcallImpl<SOME_CALL>;
+def other_impl : RuntimeLibcallImpl<OTHER_CALL>;
+
+def IsFoo : LibcallPredicate<[{TT.getArch() == Triple::arm}]>;
+def isFoo : RuntimeLibcallAvailability<(all_of IsFoo)>;
+
+// Predicate using the isAAPCS_ABI member helper.
+def IsAAPCS : LibcallPredicate<[{isAAPCS_ABI(TT, ABIName)}]>;
+def isAAPCS : RuntimeLibcallAvailability<(all_of IsAAPCS)>;
+
+// CC naming the DefaultCC sentinel.
+def FooDefaultOrAAPCS : LibcallCallingConv<
+ [{isAAPCS_ABI(TT, ABIName) ? static_cast<CallingConv::ID>(CallingConv::ARM_AAPCS) : static_cast<CallingConv::ID>(DefaultCC)}]
+>;
+
+// One member gated on isAAPCS_ABI, one whose CC names DefaultCC.
+def FooLib : LibcallLibrary<"foo", (add
+ LibcallImpls<(add some_impl), isAAPCS>,
+ LibcallsWithCC<(add other_impl), FooDefaultOrAAPCS>)>;
+
+def FooSystem : SystemRuntimeLibrary<isFoo, (add FooLib)> {
+ let DefaultLibcallCallingConv
+ = LibcallCallingConv<[{FloatABI == FloatABI::Hard ? CallingConv::ARM_AAPCS_VFP : CallingConv::ARM_AAPCS}]>;
+}
+
+// The library function seeds DefaultCC
+
+// CHECK: void llvm::RTLIB::RuntimeLibcallsInfo::setAvailableLibFuncs_foo(const llvm::Triple &TT, ExceptionHandling ExceptionModel, FloatABI::ABIType FloatABI, StringRef ABIName, LongDoubleFormat LongDoubleFormat) {
+// CHECK-NEXT: const CallingConv::ID DefaultCC = FloatABI == FloatABI::Hard ? CallingConv::ARM_AAPCS_VFP : CallingConv::ARM_AAPCS;
+
+// CHECK: setLibcallImplCallingConv(Impl, isAAPCS_ABI(TT, ABIName) ? static_cast<CallingConv::ID>(CallingConv::ARM_AAPCS) : static_cast<CallingConv::ID>(DefaultCC));
+// CHECK: if (isAAPCS_ABI(TT, ABIName)) {
+
+// The dispatcher seeds its own whole-table DefaultCC.
+
+// CHECK: void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets(
+// CHECK: const CallingConv::ID DefaultCC = FloatABI == FloatABI::Hard ? CallingConv::ARM_AAPCS_VFP : CallingConv::ARM_AAPCS;
+// CHECK: if (isLibraryAvailable("foo"))
+// CHECK-NEXT: setAvailableLibFuncs_foo(TT, ExceptionModel, FloatABI, ABIName, LongDoubleFormat);
diff --git a/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp b/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
index 55d29a9238bd27..980d474ed6ee6d 100644
--- a/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
@@ -124,10 +124,14 @@ class RuntimeLibcallEmitter {
// Emit a `setAvailableLibFuncs_<name>` member function for all LibcallLibrary
// defs sharing \p Name, each gated by its own availability predicate. \p
- // Exclusions are emitted as guarded setUnavailable calls at the end.
+ // Exclusions are emitted as guarded setUnavailable calls at the end. \p
+ // DefaultCCs holds the distinct DefaultLibcallCallingConv snippets the
+ // consuming system libraries supply; exactly one must exist if any member CC
+ // names the DefaultCC sentinel.
void emitLibraryFunction(raw_ostream &OS, StringRef Name,
ArrayRef<const Record *> Libs,
- ArrayRef<LibraryExclusion> Exclusions) const;
+ ArrayRef<LibraryExclusion> Exclusions,
+ ArrayRef<StringRef> DefaultCCs) const;
// Group all LibcallLibrary defs by their shared LibraryName, preserving
// definition order. Both the member-declaration fragment and the definitions
@@ -544,7 +548,8 @@ void RuntimeLibcallEmitter::emitLibraryVariant(raw_ostream &OS,
void RuntimeLibcallEmitter::emitLibraryFunction(
raw_ostream &OS, StringRef Name, ArrayRef<const Record *> Libs,
- ArrayRef<LibraryExclusion> Exclusions) const {
+ ArrayRef<LibraryExclusion> Exclusions,
+ ArrayRef<StringRef> DefaultCCs) const {
OS << "void llvm::RTLIB::RuntimeLibcallsInfo::setAvailableLibFuncs_";
emitLibFuncSuffix(OS, Name);
OS << "(const llvm::Triple &TT, "
@@ -600,6 +605,37 @@ void RuntimeLibcallEmitter::emitLibraryFunction(
Expanded.push_back(std::move(EL));
}
+ // If any member CC names the DefaultCC sentinel, emit a local for it (seeded
+ // from the consuming system library's DefaultLibcallCallingConv) so those
+ // snippets are in scope. \p DefaultCCs holds the distinct snippets the
+ // consumers supply: exactly one must exist.
+ bool ReferencesDefaultCC = any_of(Expanded, [](const ExpandedLibrary &EL) {
+ return any_of(EL.Pred2Funcs, [](const auto &KeyAndFuncs) {
+ const Record *CC = KeyAndFuncs.second.CallingConv;
+ return CC && CC->getValueAsString("CallingConv").contains("DefaultCC");
+ });
+ });
+
+ if (ReferencesDefaultCC) {
+ if (DefaultCCs.empty()) {
+ PrintFatalError(Libs.front(),
+ "library '" + Name +
+ "' has a member calling convention referencing "
+ "DefaultCC but no consuming SystemRuntimeLibrary "
+ "provides a DefaultLibcallCallingConv");
+ }
+ if (DefaultCCs.size() > 1) {
+ PrintFatalError(Libs.front(),
+ "library '" + Name +
+ "' is dispatched by multiple SystemRuntimeLibrary "
+ "defs with different DefaultLibcallCallingConv; "
+ "DefaultCC is ambiguous for its member calling "
+ "conventions");
+ }
+
+ OS << " const CallingConv::ID DefaultCC = " << DefaultCCs.front() << ";\n";
+ }
+
// Impls unconditional in every variant are emitted once and stripped from
// each variant, so the shared core is not repeated.
SetVector<const RuntimeLibcallImpl *> SharedCore;
@@ -735,8 +771,47 @@ void RuntimeLibcallEmitter::emitSystemRuntimeLibrarySetCalls(
}
}
- for (const auto &[Name, Libs] : collectLibrariesByName())
- emitLibraryFunction(OS, Name, Libs, ExclusionsByLibName.lookup(Name));
+ // Collect, per library name, the distinct DefaultLibcallCallingConv snippets
+ // its consuming system libraries supply (a plain Record walk; no member
+ // expansion). emitLibraryFunction, which already expands the members, picks
+ // the snippet for a library that names the DefaultCC sentinel and diagnoses a
+ // missing (none) or ambiguous (more than one) snippet.
+ MapVector<StringRef, SetVector<StringRef>> DefaultCCsByLibName;
+ for (const Record *R : AllLibs) {
+ const Record *DefaultCCClass =
+ R->getValueAsDef("DefaultLibcallCallingConv");
+ StringRef DefaultCC =
+ DefaultCCClass ? DefaultCCClass->getValueAsString("CallingConv").trim()
+ : StringRef();
+ if (DefaultCC.empty())
+ continue;
+ const DagInit *MemberDag =
+ R->getValueAsDef("MemberList")->getValueAsDag("MemberList");
+ for (const Init *Arg : MemberDag->getArgs()) {
+ const auto *DI = dyn_cast<DefInit>(Arg);
+ if (!DI)
+ continue;
+ const Record *Def = DI->getDef();
+ const Record *Lib = nullptr;
+ if (Def->isSubClassOf("LibcallLibrary"))
+ Lib = Def;
+ else if (Def->isSubClassOf("LibraryRef"))
+ Lib = Def->getValueAsDef("Library");
+ if (!Lib)
+ continue;
+ DefaultCCsByLibName[Lib->getValueAsString("LibraryName")].insert(
+ DefaultCC);
+ }
+ }
+
+ for (const auto &[Name, Libs] : collectLibrariesByName()) {
+ auto It = DefaultCCsByLibName.find(Name);
+ ArrayRef<StringRef> DefaultCCs = It == DefaultCCsByLibName.end()
+ ? ArrayRef<StringRef>()
+ : It->second.getArrayRef();
+ emitLibraryFunction(OS, Name, Libs, ExclusionsByLibName.lookup(Name),
+ DefaultCCs);
+ }
OS << "void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets("
"const llvm::Triple &TT, ExceptionHandling ExceptionModel, "
>From 3afd421427ed7ae7813133cf2fd4b694d8bec4e5 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Tue, 6 Oct 2026 22:45:14 +0100
Subject: [PATCH 35/46] [ARM] Protect against multi use increment in gather
scatter combine (#227247)
Fixes #226592
---
.../Target/ARM/MVEGatherScatterLowering.cpp | 10 +-
.../CodeGen/Thumb2/mve-gather-increment.ll | 113 ++++++++++++++----
2 files changed, 94 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
index 0f4b9e221e6079..223bc73b3f93cd 100644
--- a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
+++ b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
@@ -1045,17 +1045,9 @@ bool MVEGatherScatterLowering::optimiseOffsets(Value *Offsets, BasicBlock *BB,
// If the phi is not used by anything else, we can just adapt it when
// replacing the instruction; if it is, we'll have to duplicate it
PHINode *NewPhi;
- if (Phi->hasNUses(2)) {
+ if (Phi->hasNUses(2) && IncInstruction->hasOneUse()) {
// No other users -> reuse existing phi (One user is the instruction
// we're looking at, the other is the phi increment)
- if (!IncInstruction->hasOneUse()) {
- // If the incrementing instruction does have more users than
- // our phi, we need to copy it
- IncInstruction = BinaryOperator::Create(
- Instruction::BinaryOps(IncInstruction->getOpcode()), Phi,
- IncrementPerRound, "LoopIncrement", IncInstruction->getIterator());
- Phi->setIncomingValue(IncrementingBlock, IncInstruction);
- }
NewPhi = Phi;
} else {
// There are other users -> create a new phi
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll b/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
index c43f76ea10444c..2ba416a7dd1131 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
@@ -1590,24 +1590,97 @@ for.cond.cleanup: ; preds = %vector.body, %entry
ret void
}
+define void @_Z6gatherv() {
+; CHECK-LABEL: _Z6gatherv:
+; CHECK: @ %bb.0: @ %entry
+; CHECK-NEXT: .save {r7, lr}
+; CHECK-NEXT: push {r7, lr}
+; CHECK-NEXT: .pad #576
+; CHECK-NEXT: sub.w sp, sp, #576
+; CHECK-NEXT: mov.w lr, #30
+; CHECK-NEXT: adr r0, .LCPI20_0
+; CHECK-NEXT: add r1, sp, #96
+; CHECK-NEXT: vldrw.u32 q0, [r0]
+; CHECK-NEXT: movs r0, #4
+; CHECK-NEXT: movs r2, #1
+; CHECK-NEXT: mov r3, r1
+; CHECK-NEXT: .LBB20_1: @ %vector.body
+; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: vadd.i32 q1, q0, r0
+; CHECK-NEXT: vadd.i32 q0, q0, r2
+; CHECK-NEXT: vcvt.f32.u32 q0, q0
+; CHECK-NEXT: vstrb.8 q0, [r3], #16
+; CHECK-NEXT: vmov q0, q1
+; CHECK-NEXT: le lr, .LBB20_1
+; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT: adr r2, .LCPI20_1
+; CHECK-NEXT: mov.w lr, #6
+; CHECK-NEXT: vldrw.u32 q0, [r2]
+; CHECK-NEXT: mov r0, sp
+; CHECK-NEXT: movs r2, #0
+; CHECK-NEXT: vadd.i32 q0, q0, r1
+; CHECK-NEXT: movs r1, #4
+; CHECK-NEXT: .LBB20_3: @ %for.cond6.preheader
+; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: vdup.32 q1, r2
+; CHECK-NEXT: add r2, r1
+; CHECK-NEXT: vshl.i32 q1, q1, #2
+; CHECK-NEXT: vadd.i32 q1, q0, q1
+; CHECK-NEXT: vldrw.u32 q2, [q1]
+; CHECK-NEXT: vstrb.8 q2, [r0], #16
+; CHECK-NEXT: le lr, .LBB20_3
+; CHECK-NEXT: @ %bb.4: @ %for.cond.cleanup3
+; CHECK-NEXT: add.w sp, sp, #576
+; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: @ %bb.5:
+; CHECK-NEXT: .LCPI20_0:
+; CHECK-NEXT: .long 0 @ 0x0
+; CHECK-NEXT: .long 1 @ 0x1
+; CHECK-NEXT: .long 2 @ 0x2
+; CHECK-NEXT: .long 3 @ 0x3
+; CHECK-NEXT: .LCPI20_1:
+; CHECK-NEXT: .long 12 @ 0xc
+; CHECK-NEXT: .long 4 @ 0x4
+; CHECK-NEXT: .long 0 @ 0x0
+; CHECK-NEXT: .long 8 @ 0x8
+entry:
+ %input = alloca [120 x float], align 4
+ %out = alloca [24 x float], align 4
+ call void @llvm.lifetime.start.p0(ptr nonnull %input) #4
+ br label %vector.body
-declare <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i32>)
-declare <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i32>)
-declare <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x i32>)
-declare <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x i32>)
-declare <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x float>)
-declare <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x float>)
-declare <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x float>)
-declare <2 x i16> @llvm.masked.gather.v2i16.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i16>)
-declare <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i16>)
-declare <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x i16>)
-declare <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x i16>)
-declare <4 x half> @llvm.masked.gather.v4f16.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x half>)
-declare <8 x half> @llvm.masked.gather.v8f16.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x half>)
-declare <16 x half> @llvm.masked.gather.v16f16.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x half>)
-declare <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i8>)
-declare <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x i8>)
-declare <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x i8>)
-declare <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr>, i32, <32 x i1>, <32 x i8>)
-declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32, <4 x i1>)
-declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
+vector.body: ; preds = %vector.body, %entry
+ %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.ind = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %entry ], [ %vec.ind.next, %vector.body ]
+ %0 = add nuw nsw <4 x i32> %vec.ind, splat (i32 1)
+ %1 = uitofp nneg <4 x i32> %0 to <4 x float>
+ %2 = getelementptr inbounds nuw [4 x i8], ptr %input, i32 %index
+ store <4 x float> %1, ptr %2, align 4
+ %index.next = add nuw i32 %index, 4
+ %vec.ind.next = add nuw nsw <4 x i32> %vec.ind, splat (i32 4)
+ %3 = icmp eq i32 %index.next, 120
+ br i1 %3, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup: ; preds = %vector.body
+ call void @llvm.lifetime.start.p0(ptr nonnull %out) #4
+ %invariant.gep = getelementptr [4 x i8], ptr %input, <4 x i32> <i32 3, i32 1, i32 0, i32 2>
+ br label %for.cond6.preheader
+
+for.cond6.preheader: ; preds = %for.cond.cleanup, %for.cond6.preheader
+ %outer.033 = phi i32 [ 0, %for.cond.cleanup ], [ %inc20, %for.cond6.preheader ]
+ %mul = shl nuw nsw i32 %outer.033, 2
+ %4 = getelementptr inbounds nuw [4 x i8], ptr %out, i32 %mul
+ %gep = getelementptr [4 x i8], <4 x ptr> %invariant.gep, i32 %mul
+ %wide.masked.gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 %gep, <4 x i1> splat (i1 true), <4 x float> poison)
+ store <4 x float> %wide.masked.gather, ptr %4, align 4
+ %inc20 = add nuw nsw i32 %outer.033, 1
+ %exitcond35.not = icmp eq i32 %inc20, 6
+ br i1 %exitcond35.not, label %for.cond.cleanup3, label %for.cond6.preheader
+
+for.cond.cleanup3: ; preds = %for.cond6.preheader
+ store i32 24, ptr %out
+ call void @llvm.lifetime.end.p0(ptr nonnull %out) #4
+ call void @llvm.lifetime.end.p0(ptr nonnull %input) #4
+ ret void
+}
>From aa80e4117280e9bb25187f92aef1f8dc50341d56 Mon Sep 17 00:00:00 2001
From: Alan Li <me at alanli.org>
Date: Tue, 6 Oct 2026 17:45:59 -0400
Subject: [PATCH 36/46] [AMDGPU] Account for gfx950 permlane hazards during
scheduling (#229517)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The [CDNA4 ISA Reference Guide, §4.5, Table 11, p.
21](https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/instruction-set-architectures/amd-instinct-cdna4-instruction-set-architecture.pdf#page=29)
requires 2 wait states between a VALU writing a VGPR and a permlane
reading it, and 4 between a `VCMPX` writing EXEC and a permlane. As far
as I know, this is GFX950 specific.
Right now the gfx950 scheduler does not query this existing hazard
check, so final hazard processing can insert avoidable `nop`s despite
available independent instructions.
This patch expose the existing permlane hazard check in
`getHazardType()`.
Also add MIR coverage for source dependencies, unrelated writes,
available fillers, and VCMPX scheduling boundaries.
Partially addresses: #228816
---
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 6 +-
llvm/lib/Target/AMDGPU/GCNSubtarget.h | 2 +
.../CodeGen/AMDGPU/postra-permlane-hazard.mir | 127 ++++++++++++++++++
3 files changed, 134 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/postra-permlane-hazard.mir
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index cf3c260cf6494e..83ba39c0df1a33 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -632,6 +632,10 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
checkMAILdStHazards(MI) > 0)
return HazardType;
+ if (ST.hasPermlaneForwardingHazard() && isPermlane(*MI) &&
+ checkPermlaneHazards(MI) > 0)
+ return HazardType;
+
if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
return HazardType;
@@ -787,7 +791,7 @@ unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) const {
if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI))
return std::max(WaitStates, checkMAILdStHazards(MI));
- if (ST.hasGFX950Insts() && isPermlane(*MI))
+ if (ST.hasPermlaneForwardingHazard() && isPermlane(*MI))
return std::max(WaitStates, checkPermlaneHazards(MI));
return WaitStates;
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 28e57bcf6a1c06..dee1d5d0120ed9 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -626,6 +626,8 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
bool hasCvtScaleForwardingHazard() const { return HasGFX950Insts; }
+ bool hasPermlaneForwardingHazard() const { return HasGFX950Insts; }
+
// All GFX9 targets experience a fetch delay when an instruction at the start
// of a loop header is split by a 32-byte fetch window boundary, but GFX950
// is uniquely sensitive to this: the delay triggers further performance
diff --git a/llvm/test/CodeGen/AMDGPU/postra-permlane-hazard.mir b/llvm/test/CodeGen/AMDGPU/postra-permlane-hazard.mir
new file mode 100644
index 00000000000000..7228bd773b5807
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/postra-permlane-hazard.mir
@@ -0,0 +1,127 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=postmisched -verify-machineinstrs -verify-misched %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=postmisched,post-RA-hazard-rec -verify-machineinstrs -verify-misched %s -o - | FileCheck %s --check-prefixes=CHECK,FINAL
+
+# A permlane reading a VALU result needs two intervening wait states on gfx950.
+# The scheduler should use available independent instructions for these slots,
+# rather than leave them after the permlane for the hazard pass to insert NOPs.
+
+---
+name: valu_vdst0_two_fillers
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: valu_vdst0_two_fillers
+ ; CHECK: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $sgpr0 = S_MOV_B32 0
+ ; CHECK-NEXT: $sgpr1 = S_MOV_B32 1
+ ; CHECK-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $vgpr1
+
+ $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
+ $sgpr0 = S_MOV_B32 0
+ $sgpr1 = S_MOV_B32 1
+ S_ENDPGM 0
+...
+
+# Check the other explicit source and a VOP3 permlane32 swap.
+
+---
+name: valu_vdst1_two_fillers
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: valu_vdst1_two_fillers
+ ; CHECK: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $sgpr0 = S_MOV_B32 0
+ ; CHECK-NEXT: $sgpr1 = S_MOV_B32 1
+ ; CHECK-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $vgpr0
+
+ $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+ renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
+ $sgpr0 = S_MOV_B32 0
+ $sgpr1 = S_MOV_B32 1
+ S_ENDPGM 0
+...
+
+# A write to a register that the permlane does not read needs no spacing.
+
+---
+name: unrelated_valu_write
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: unrelated_valu_write
+ ; CHECK: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+
+ $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
+ S_ENDPGM 0
+...
+
+# Scheduler stalls do not emit instructions. With no fillers, the hazard pass
+# must still insert the required NOPs in the final instruction stream.
+
+---
+name: valu_without_fillers
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: valu_without_fillers
+ ; CHECK: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; FINAL-NEXT: S_NOP 1
+ ; CHECK-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $vgpr1
+
+ $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
+ S_ENDPGM 0
+...
+
+# One independent instruction fills one wait state; one NOP remains necessary.
+
+---
+name: valu_with_one_filler
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: valu_with_one_filler
+ ; CHECK: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $sgpr0 = S_MOV_B32 0
+ ; FINAL-NEXT: S_NOP 0
+ ; CHECK-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $vgpr1
+
+ $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
+ $sgpr0 = S_MOV_B32 0
+ S_ENDPGM 0
+...
+
+# VCMPX writes EXEC and forms a scheduling boundary. Its four-wait-state hazard
+# must still be handled by the final hazard pass across that boundary.
+
+---
+name: vcmpx_without_fillers
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: vcmpx_without_fillers
+ ; CHECK: V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
+ ; FINAL-NEXT: S_NOP 3
+ ; CHECK-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+
+ V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
+ renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
+ S_ENDPGM 0
+...
>From 1c693d7cea55277eeddb068d7b02e37ceaacac31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
=?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
=?UTF-8?q?=E3=83=B3=29?= <clementval at gmail.com>
Date: Tue, 6 Oct 2026 14:52:28 -0700
Subject: [PATCH 37/46] [flang-rt][cuda] Add CUFRegisterHostMemoryRange entry
point (#229572)
Extracted runtime part from #229213
---
flang-rt/lib/cuda/registration.cpp | 25 +++++++++++++++++++
.../include/flang/Runtime/CUDA/registration.h | 5 ++++
2 files changed, 30 insertions(+)
diff --git a/flang-rt/lib/cuda/registration.cpp b/flang-rt/lib/cuda/registration.cpp
index 1910b149653a9d..ec93a061cf716e 100644
--- a/flang-rt/lib/cuda/registration.cpp
+++ b/flang-rt/lib/cuda/registration.cpp
@@ -11,6 +11,8 @@
#include "flang/Runtime/CUDA/common.h"
#include "cuda_runtime.h"
+#include <cstdint>
+#include <unistd.h>
namespace Fortran::runtime::cuda {
@@ -55,6 +57,29 @@ void RTDEF(CUFRegisterManagedVariable)(
void RTDEF(CUFInitModule)(void **module) { __cudaInitModule(module); }
+void RTDEF(CUFRegisterHostMemoryRange)(void *begin, void *end) {
+ if (!begin || end <= begin)
+ return;
+ const auto pageSize{static_cast<std::uintptr_t>(sysconf(_SC_PAGESIZE))};
+ const auto first{reinterpret_cast<std::uintptr_t>(begin) & ~(pageSize - 1)};
+ const auto last{
+ (reinterpret_cast<std::uintptr_t>(end) + pageSize - 1) & ~(pageSize - 1)};
+ cudaError_t err{cudaHostRegister(reinterpret_cast<void *>(first),
+ last - first, cudaHostRegisterPortable | cudaHostRegisterMapped)};
+ if (err == cudaErrorHostMemoryAlreadyRegistered) {
+ // Clear the error state left by the failed call.
+ (void)cudaGetLastError();
+ return;
+ }
+ if (err != cudaSuccess) {
+ const char *name{cudaGetErrorName(err)};
+ Terminator terminator{__FILE__, __LINE__};
+ terminator.Crash("cudaHostRegister(%p, %zu) failed with '%s'",
+ reinterpret_cast<void *>(first), static_cast<std::size_t>(last - first),
+ name ? name : "<unknown>");
+ }
+}
+
} // extern "C"
} // namespace Fortran::runtime::cuda
diff --git a/flang/include/flang/Runtime/CUDA/registration.h b/flang/include/flang/Runtime/CUDA/registration.h
index 74dbf9e1890768..9286d7173adc8e 100644
--- a/flang/include/flang/Runtime/CUDA/registration.h
+++ b/flang/include/flang/Runtime/CUDA/registration.h
@@ -37,6 +37,11 @@ void RTDECL(CUFRegisterManagedVariable)(
/// unified memory addresses.
void RTDECL(CUFInitModule)(void **module);
+/// Register the pages spanning [\p begin, \p end) with the CUDA runtime so
+/// device code can access them through their host address. Registering the
+/// same range more than once is allowed.
+void RTDECL(CUFRegisterHostMemoryRange)(void *begin, void *end);
+
} // extern "C"
} // namespace Fortran::runtime::cuda
>From 3f4c27d3c4444059981c56437a5efe9cd6c32385 Mon Sep 17 00:00:00 2001
From: Alexey Samsonov <vonosmas at gmail.com>
Date: Tue, 6 Oct 2026 14:53:16 -0700
Subject: [PATCH 38/46] [libc][locale] Fix nullptr argument handling in
setlocale. (#229569)
`setlocale` function should accept `nullptr` as a possible value for the
second argument - in this case the
locale is not modified but queried. Make sure to return `C` locale in
this case (the only one supported by LLVM-libc).
Expand the unit test to verify `setlocale` behavior.
---
libc/src/locale/newlocale.h | 13 +++++++++----
libc/src/locale/setlocale.cpp | 16 +++++++++++++---
libc/test/src/locale/CMakeLists.txt | 1 +
libc/test/src/locale/locale_test.cpp | 25 ++++++++++++++++++++++++-
4 files changed, 47 insertions(+), 8 deletions(-)
diff --git a/libc/src/locale/newlocale.h b/libc/src/locale/newlocale.h
index 07972fa384433c..211bbb19b9f8c7 100644
--- a/libc/src/locale/newlocale.h
+++ b/libc/src/locale/newlocale.h
@@ -1,13 +1,18 @@
-//===-- Implementation header for setlocale ---------------------*- C++ -*-===//
+//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Implementation header for newlocale.
+///
+//===----------------------------------------------------------------------===//
-#ifndef LLVM_LIBC_SRC_LOCALE_SETLOCALE_H
-#define LLVM_LIBC_SRC_LOCALE_SETLOCALE_H
+#ifndef LLVM_LIBC_SRC_LOCALE_NEWLOCALE_H
+#define LLVM_LIBC_SRC_LOCALE_NEWLOCALE_H
#include "hdr/types/locale_t.h"
#include "src/__support/macros/config.h"
@@ -18,4 +23,4 @@ locale_t newlocale(int category_mask, const char *locale_name, locale_t base);
} // namespace LIBC_NAMESPACE_DECL
-#endif // LLVM_LIBC_SRC_LOCALE_SETLOCALE_H
+#endif // LLVM_LIBC_SRC_LOCALE_NEWLOCALE_H
diff --git a/libc/src/locale/setlocale.cpp b/libc/src/locale/setlocale.cpp
index 2dec497ce051a4..bdc311eead2a7e 100644
--- a/libc/src/locale/setlocale.cpp
+++ b/libc/src/locale/setlocale.cpp
@@ -1,10 +1,15 @@
-//===-- Implementation of setlocale ---------------------------------------===//
+//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Implementation of setlocale.
+///
+//===----------------------------------------------------------------------===//
#include "src/locale/setlocale.h"
#include "hdr/locale_macros.h"
@@ -15,10 +20,15 @@
namespace LIBC_NAMESPACE_DECL {
LLVM_LIBC_FUNCTION(char *, setlocale, (int category, const char *locale_name)) {
- cpp::string_view name(locale_name);
- if (category > LC_ALL || (!name.empty() && name != "C"))
+ if (category > LC_ALL)
return nullptr;
+ if (locale_name != nullptr) {
+ cpp::string_view name(locale_name);
+ if (!name.empty() && name != "C")
+ return nullptr;
+ }
+
static char locale_str[] = "C";
return locale_str;
}
diff --git a/libc/test/src/locale/CMakeLists.txt b/libc/test/src/locale/CMakeLists.txt
index 5032ce9c324e23..2a50e2ed87d1b9 100644
--- a/libc/test/src/locale/CMakeLists.txt
+++ b/libc/test/src/locale/CMakeLists.txt
@@ -9,6 +9,7 @@ add_libc_test(
DEPENDS
libc.hdr.locale_macros
libc.src.locale.newlocale
+ libc.src.locale.setlocale
libc.src.locale.uselocale
libc.src.locale.freelocale
)
diff --git a/libc/test/src/locale/locale_test.cpp b/libc/test/src/locale/locale_test.cpp
index d19e2605fca375..a55ad85006260c 100644
--- a/libc/test/src/locale/locale_test.cpp
+++ b/libc/test/src/locale/locale_test.cpp
@@ -1,14 +1,20 @@
-//===-- Unittests for locale ----------------------------------------------===//
+//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unittests for locale.
+///
+//===----------------------------------------------------------------------===//
#include "hdr/locale_macros.h"
#include "src/locale/freelocale.h"
#include "src/locale/newlocale.h"
+#include "src/locale/setlocale.h"
#include "src/locale/uselocale.h"
#include "test/UnitTest/Test.h"
@@ -45,3 +51,20 @@ TEST(LlvmLibcLocale, NewLocaleValidation) {
loc = LIBC_NAMESPACE::newlocale(LC_ALL_MASK, "does-not-exist", nullptr);
EXPECT_EQ(loc, static_cast<locale_t>(nullptr));
}
+
+TEST(LlvmLibcLocale, SetLocale) {
+ // Setting a "C" or default locale.
+ EXPECT_STREQ(LIBC_NAMESPACE::setlocale(LC_ALL, "C"), "C");
+ EXPECT_STREQ(LIBC_NAMESPACE::setlocale(LC_CTYPE, ""), "C");
+ // Querying the current locale.
+ EXPECT_STREQ(LIBC_NAMESPACE::setlocale(LC_ALL, nullptr), "C");
+ EXPECT_STREQ(LIBC_NAMESPACE::setlocale(LC_NUMERIC, nullptr), "C");
+
+ // Unsupporter categories or non-C locales.
+ EXPECT_EQ(LIBC_NAMESPACE::setlocale(LC_ALL + 1, "C"),
+ static_cast<char *>(nullptr));
+ EXPECT_EQ(LIBC_NAMESPACE::setlocale(LC_ALL + 1, nullptr),
+ static_cast<char *>(nullptr));
+ EXPECT_EQ(LIBC_NAMESPACE::setlocale(LC_ALL, "does-not-exist"),
+ static_cast<char *>(nullptr));
+}
>From c99f5ac58eab06359dd50dafa39eb1ba382cf88c Mon Sep 17 00:00:00 2001
From: Jeong Jihyeon <84846295+JihyeonJeong129 at users.noreply.github.com>
Date: Wed, 7 Oct 2026 06:53:56 +0900
Subject: [PATCH 39/46] [libc] Add 'x' and 'e' mode support to fopen()
(#224207)
Add exclusive creation (x) and close-on-exec (e) support to the fopen.
Also extended the related close-on-exec handling to fdopen() and to
freopen()
Fixes #223070
Assisted-by: Codex
---
libc/src/__support/File/file_mode.h | 14 ++-
libc/src/__support/File/linux/CMakeLists.txt | 1 +
libc/src/__support/File/linux/file.cpp | 33 ++++++-
libc/src/__support/File/linux/file_flags.h | 2 +
.../src/__support/File/file_mode_test.cpp | 47 +++++++++
libc/test/src/stdio/CMakeLists.txt | 18 ++++
libc/test/src/stdio/fdopen_test.cpp | 40 ++++++++
libc/test/src/stdio/fopen_test.cpp | 95 +++++++++++++++++++
libc/test/src/stdio/freopen_test.cpp | 35 +++++++
9 files changed, 282 insertions(+), 3 deletions(-)
diff --git a/libc/src/__support/File/file_mode.h b/libc/src/__support/File/file_mode.h
index f9aece8d22a36d..8049516da261f0 100644
--- a/libc/src/__support/File/file_mode.h
+++ b/libc/src/__support/File/file_mode.h
@@ -62,6 +62,9 @@ class FileMode {
case 'x':
file_mode_ |= static_cast<Mode>(CreateType::EXCLUSIVE);
break;
+ case 'e':
+ file_mode_ |= static_cast<Mode>(Extension::CLOSE_ON_EXEC);
+ break;
default:
file_mode_ = 0;
}
@@ -108,6 +111,10 @@ class FileMode {
return (file_mode_ & static_cast<Mode>(CreateType::EXCLUSIVE)) != 0;
}
+ constexpr bool is_close_on_exec() const {
+ return (file_mode_ & static_cast<Mode>(Extension::CLOSE_ON_EXEC)) != 0;
+ }
+
private:
// Mode is a generic or abstract mode bit for all kinds of modes
// (open-mode, 'content-mode', 'create-modes')
@@ -115,7 +122,7 @@ class FileMode {
// Denotes the mode of the file.
//
- // The three different types of flags below are to be used with '|' operator.
+ // The different types of flags below are to be used with '|' operator.
// Their values correspond to mutually exclusive bits in a 32-bit unsigned
// integer value. A flag set can include both READ and WRITE if the file
// is opened in update mode (ie. if the file was opened with a '+' the mode
@@ -138,6 +145,11 @@ class FileMode {
EXCLUSIVE = 0x100,
};
+ // POSIX extensions to the ISO C file modes.
+ enum class Extension : Mode {
+ CLOSE_ON_EXEC = 0x1000,
+ };
+
// This property tracks the mode for the particular file instance (i.e
// currently opened file)
Mode file_mode_;
diff --git a/libc/src/__support/File/linux/CMakeLists.txt b/libc/src/__support/File/linux/CMakeLists.txt
index d0e1df252895c1..05efc0f6f61395 100644
--- a/libc/src/__support/File/linux/CMakeLists.txt
+++ b/libc/src/__support/File/linux/CMakeLists.txt
@@ -14,6 +14,7 @@ add_object_library(
libc.hdr.types.off_t
libc.src.__support.CPP.new
libc.src.__support.OSUtil.linux.syscall_wrappers.close
+ libc.src.__support.OSUtil.linux.syscall_wrappers.dup3
libc.src.__support.OSUtil.linux.syscall_wrappers.fcntl
libc.src.__support.OSUtil.linux.syscall_wrappers.lseek
libc.src.__support.OSUtil.linux.syscall_wrappers.open
diff --git a/libc/src/__support/File/linux/file.cpp b/libc/src/__support/File/linux/file.cpp
index cc6a1fcbd31688..1d360c15bfa834 100644
--- a/libc/src/__support/File/linux/file.cpp
+++ b/libc/src/__support/File/linux/file.cpp
@@ -15,7 +15,7 @@
#include "src/__support/File/file.h"
#include "src/__support/File/file_mode.h"
#include "src/__support/OSUtil/linux/syscall_wrappers/close.h"
-#include "src/__support/OSUtil/linux/syscall_wrappers/dup2.h"
+#include "src/__support/OSUtil/linux/syscall_wrappers/dup3.h"
#include "src/__support/OSUtil/linux/syscall_wrappers/fcntl.h"
#include "src/__support/OSUtil/linux/syscall_wrappers/lseek.h"
#include "src/__support/OSUtil/linux/syscall_wrappers/open.h"
@@ -87,6 +87,14 @@ static int map_c_mode_flags_to_linux_open_flags(const FileMode &file_mode) {
else if (file_mode.is_write())
open_flags |= LinuxFileFlags::CREATE_OR_TRUNCATE;
+ // Ignore 'x' for read modes: O_EXCL is only meaningful with O_CREAT here.
+ if (file_mode.is_exclusive_create() &&
+ (file_mode.is_write() || file_mode.is_append()))
+ open_flags |= LinuxFileFlags::EXCLUSIVE_CREATE;
+
+ if (file_mode.is_close_on_exec())
+ open_flags |= LinuxFileFlags::CLOSE_ON_EXEC;
+
return open_flags;
}
@@ -119,6 +127,14 @@ ErrorOr<File *> openfile(const char *path, const char *mode) {
return file;
}
+static ErrorOr<int> set_close_on_exec(int fd) {
+ auto flags = linux_syscalls::fcntl(fd, F_GETFD);
+ if (!flags)
+ return Error(flags.error());
+ return linux_syscalls::fcntl(
+ fd, F_SETFD, reinterpret_cast<void *>(flags.value() | FD_CLOEXEC));
+}
+
ErrorOr<LinuxFile *> create_file_from_fd(int fd, const char *mode) {
const FileMode file_mode(mode);
@@ -139,6 +155,12 @@ ErrorOr<LinuxFile *> create_file_from_fd(int fd, const char *mode) {
return Error(EINVAL);
}
+ if (file_mode.is_close_on_exec()) {
+ auto cloexec_result = set_close_on_exec(fd);
+ if (!cloexec_result)
+ return Error(cloexec_result.error());
+ }
+
bool do_seek = false;
if (file_mode.is_append() &&
!LinuxFileFlags::file_has_append_flag(fd_flags)) {
@@ -195,6 +217,10 @@ int LinuxFile::reopen_unlocked(const char *path, const char *mode) {
int open_flags = map_c_mode_flags_to_linux_open_flags(file_mode);
+ // Prevent the temporary descriptor from being inherited across exec.
+ if (old_fd >= 0)
+ open_flags |= LinuxFileFlags::CLOSE_ON_EXEC;
+
ErrorOr<int> new_fd =
linux_syscalls::open(path, open_flags, LinuxFileFlags::OPEN_MODE);
@@ -215,7 +241,10 @@ int LinuxFile::reopen_unlocked(const char *path, const char *mode) {
// Else the new file successfully opened, so we move it into the fd the old
// file was using if the old fd exists.
if (old_fd >= 0) {
- auto dup_result = linux_syscalls::dup2(new_fd.value(), old_fd);
+ // Preserve close-on-exec atomically when replacing the old descriptor.
+ int dup_flags =
+ file_mode.is_close_on_exec() ? LinuxFileFlags::CLOSE_ON_EXEC : 0;
+ auto dup_result = linux_syscalls::dup3(new_fd.value(), old_fd, dup_flags);
if (!dup_result) {
linux_syscalls::close(new_fd.value());
reset_stream_state_unlocked(file_mode);
diff --git a/libc/src/__support/File/linux/file_flags.h b/libc/src/__support/File/linux/file_flags.h
index deb357c261d556..ce68496101639b 100644
--- a/libc/src/__support/File/linux/file_flags.h
+++ b/libc/src/__support/File/linux/file_flags.h
@@ -27,6 +27,8 @@ class LinuxFileFlags {
static constexpr int WRITE_ONLY = O_WRONLY;
static constexpr int READ_ONLY = O_RDONLY;
static constexpr int CREATE_OR_TRUNCATE = O_CREAT | O_TRUNC;
+ static constexpr int EXCLUSIVE_CREATE = O_EXCL;
+ static constexpr int CLOSE_ON_EXEC = O_CLOEXEC;
// File created will have 0666 permissions.
LIBC_INLINE static constexpr mode_t OPEN_MODE =
diff --git a/libc/test/src/__support/File/file_mode_test.cpp b/libc/test/src/__support/File/file_mode_test.cpp
index 92f58bd154ee31..6048b49c8b27a5 100644
--- a/libc/test/src/__support/File/file_mode_test.cpp
+++ b/libc/test/src/__support/File/file_mode_test.cpp
@@ -47,6 +47,11 @@ TEST(LlvmLibcFileModeTest, FirstCharacterMustBeAValidMode) {
constexpr FileMode exclusive_create("x");
EXPECT_FALSE(exclusive_create.is_valid());
EXPECT_FALSE(exclusive_create.is_exclusive_create());
+
+ // close-on-exec set as first character => invalid
+ constexpr FileMode close_on_exec("e");
+ EXPECT_FALSE(close_on_exec.is_valid());
+ EXPECT_FALSE(close_on_exec.is_close_on_exec());
}
TEST(LlvmLibcFileModeTest, AllPossibleValidCombinations) {
@@ -59,6 +64,7 @@ TEST(LlvmLibcFileModeTest, AllPossibleValidCombinations) {
EXPECT_TRUE(readonly.is_valid());
EXPECT_TRUE(readonly.is_read());
EXPECT_TRUE(readonly.read_allowed());
+ EXPECT_FALSE(readonly.is_close_on_exec());
// b. Read and Update mode
constexpr FileMode read_and_update("r+");
@@ -91,6 +97,7 @@ TEST(LlvmLibcFileModeTest, AllPossibleValidCombinations) {
EXPECT_TRUE(writeonly.is_valid());
EXPECT_TRUE(writeonly.is_write());
EXPECT_TRUE(writeonly.write_allowed());
+ EXPECT_FALSE(writeonly.is_close_on_exec());
// b. Write and Update mode
constexpr FileMode write_and_update("w+");
@@ -157,6 +164,7 @@ TEST(LlvmLibcFileModeTest, AllPossibleValidCombinations) {
EXPECT_TRUE(appendonly.is_valid());
EXPECT_TRUE(appendonly.is_append());
EXPECT_TRUE(appendonly.write_allowed());
+ EXPECT_FALSE(appendonly.is_close_on_exec());
// b. Append and Update
constexpr FileMode append_and_update("a+");
@@ -183,6 +191,45 @@ TEST(LlvmLibcFileModeTest, AllPossibleValidCombinations) {
EXPECT_TRUE(append_update_binary.read_allowed());
}
+TEST(LlvmLibcFileModeTest, CloseOnExecCombinations) {
+ // a. Read Close-on-exec
+ constexpr FileMode read_close_on_exec("re");
+ EXPECT_TRUE(read_close_on_exec.is_valid());
+ EXPECT_TRUE(read_close_on_exec.is_read());
+ EXPECT_TRUE(read_close_on_exec.is_close_on_exec());
+ EXPECT_TRUE(read_close_on_exec.read_allowed());
+ EXPECT_FALSE(read_close_on_exec.write_allowed());
+ EXPECT_FALSE(read_close_on_exec.is_binary_format());
+ EXPECT_FALSE(read_close_on_exec.is_exclusive_create());
+
+ // b. Write Close-on-exec
+ constexpr FileMode write_close_on_exec("we");
+ EXPECT_TRUE(write_close_on_exec.is_valid());
+ EXPECT_TRUE(write_close_on_exec.is_write());
+ EXPECT_TRUE(write_close_on_exec.is_close_on_exec());
+ EXPECT_TRUE(write_close_on_exec.write_allowed());
+ EXPECT_FALSE(write_close_on_exec.read_allowed());
+
+ // c. Append Close-on-exec
+ constexpr FileMode append_close_on_exec("ae");
+ EXPECT_TRUE(append_close_on_exec.is_valid());
+ EXPECT_TRUE(append_close_on_exec.is_append());
+ EXPECT_TRUE(append_close_on_exec.is_close_on_exec());
+ EXPECT_TRUE(append_close_on_exec.write_allowed());
+ EXPECT_FALSE(append_close_on_exec.read_allowed());
+
+ // d. Write Close-on-exec Exclusive Update Binary
+ constexpr FileMode write_all_modifiers("wex+b");
+ EXPECT_TRUE(write_all_modifiers.is_valid());
+ EXPECT_TRUE(write_all_modifiers.is_write());
+ EXPECT_TRUE(write_all_modifiers.is_close_on_exec());
+ EXPECT_TRUE(write_all_modifiers.is_exclusive_create());
+ EXPECT_TRUE(write_all_modifiers.is_update());
+ EXPECT_TRUE(write_all_modifiers.is_binary_format());
+ EXPECT_TRUE(write_all_modifiers.read_allowed());
+ EXPECT_TRUE(write_all_modifiers.write_allowed());
+}
+
TEST(LlvmLibcFileModeTest, InvalidCombinations) {
// 1. Read and Write as main modes
constexpr FileMode read_and_write("rw");
diff --git a/libc/test/src/stdio/CMakeLists.txt b/libc/test/src/stdio/CMakeLists.txt
index d902f465d4f86e..6d22f8934e3bc0 100644
--- a/libc/test/src/stdio/CMakeLists.txt
+++ b/libc/test/src/stdio/CMakeLists.txt
@@ -10,6 +10,7 @@ add_libc_test(
libc.include.stdio
libc.hdr.fcntl_macros
libc.hdr.stdio_macros
+ libc.src.__support.CPP.scope
libc.src.__support.macros.properties.os
libc.src.errno.errno
libc.src.fcntl.fcntl
@@ -482,6 +483,19 @@ add_libc_test(
libc.src.stdio.stderr
)
+set(fopen_test_deps)
+if(LIBC_TARGET_OS STREQUAL "linux")
+ list(APPEND fopen_test_deps
+ libc.hdr.fcntl_macros
+ libc.src.errno.errno
+ libc.src.fcntl.fcntl
+ libc.src.stdio.fileno
+ libc.src.stdio.remove
+ libc.test.UnitTest.ErrnoCheckingTest
+ libc.test.UnitTest.ErrnoSetterMatcher
+ )
+endif()
+
add_libc_test(
fopen_test
SUITE
@@ -489,7 +503,9 @@ add_libc_test(
SRCS
fopen_test.cpp
DEPENDS
+ ${fopen_test_deps}
libc.src.__support.CPP.scope
+ libc.src.__support.macros.properties.os
libc.src.stdio.fread
libc.src.stdio.fwrite
libc.src.stdio.fclose
@@ -578,6 +594,8 @@ if(${LIBC_TARGET_OS} STREQUAL "linux")
DEPENDS
libc.hdr.fcntl_macros
libc.hdr.sys_stat_macros
+ libc.src.__support.CPP.scope
+ libc.src.fcntl.fcntl
libc.src.fcntl.open
libc.src.stdio.fclose
libc.src.stdio.fdopen
diff --git a/libc/test/src/stdio/fdopen_test.cpp b/libc/test/src/stdio/fdopen_test.cpp
index 0a5e9863055aa5..b6fda6afd6af79 100644
--- a/libc/test/src/stdio/fdopen_test.cpp
+++ b/libc/test/src/stdio/fdopen_test.cpp
@@ -10,6 +10,8 @@
#include "hdr/fcntl_macros.h"
#include "hdr/sys_stat_macros.h" // For S_IRWXU
+#include "src/__support/CPP/scope.h"
+#include "src/fcntl/fcntl.h"
#include "src/fcntl/open.h"
#include "src/stdio/fclose.h"
#include "src/stdio/fgets.h"
@@ -84,3 +86,41 @@ TEST_F(LlvmLibcStdioFdopenTest, InvalidMode) {
LIBC_NAMESPACE::close(fd);
ASSERT_ERRNO_SUCCESS();
}
+
+TEST_F(LlvmLibcStdioFdopenTest, CloseOnExecPreservedWithoutModifier) {
+ using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
+ constexpr const char *TEST_FILE_NAME =
+ "testdata/close_on_exec_preserved.test";
+ auto TEST_FILE = libc_make_test_file_path(TEST_FILE_NAME);
+ int fd =
+ LIBC_NAMESPACE::open(TEST_FILE, O_CREAT | O_WRONLY | O_CLOEXEC, S_IRWXU);
+ ASSERT_GE(fd, 0);
+
+ FILE *file = LIBC_NAMESPACE::fdopen(fd, "w");
+ LIBC_NAMESPACE::cpp::scope_exit close_file([&] {
+ if (file != nullptr)
+ EXPECT_THAT(LIBC_NAMESPACE::fclose(file), Succeeds(0));
+ else
+ EXPECT_THAT(LIBC_NAMESPACE::close(fd), Succeeds(0));
+ });
+ ASSERT_NE(file, nullptr);
+ EXPECT_THAT(LIBC_NAMESPACE::fcntl(fd, F_GETFD), Succeeds(FD_CLOEXEC));
+}
+
+TEST_F(LlvmLibcStdioFdopenTest, CloseOnExecEnabledByModifier) {
+ using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
+ constexpr const char *TEST_FILE_NAME = "testdata/close_on_exec_enabled.test";
+ auto TEST_FILE = libc_make_test_file_path(TEST_FILE_NAME);
+ int fd = LIBC_NAMESPACE::open(TEST_FILE, O_CREAT | O_WRONLY, S_IRWXU);
+ ASSERT_GE(fd, 0);
+
+ FILE *file = LIBC_NAMESPACE::fdopen(fd, "we");
+ LIBC_NAMESPACE::cpp::scope_exit close_file([&] {
+ if (file != nullptr)
+ EXPECT_THAT(LIBC_NAMESPACE::fclose(file), Succeeds(0));
+ else
+ EXPECT_THAT(LIBC_NAMESPACE::close(fd), Succeeds(0));
+ });
+ ASSERT_NE(file, nullptr);
+ EXPECT_THAT(LIBC_NAMESPACE::fcntl(fd, F_GETFD), Succeeds(FD_CLOEXEC));
+}
diff --git a/libc/test/src/stdio/fopen_test.cpp b/libc/test/src/stdio/fopen_test.cpp
index 249ef1356ab71c..48a7b4ba9161d3 100644
--- a/libc/test/src/stdio/fopen_test.cpp
+++ b/libc/test/src/stdio/fopen_test.cpp
@@ -8,6 +8,7 @@
#include "src/__support/CPP/scope.h"
#include "src/__support/File/file.h"
+#include "src/__support/macros/properties/os.h"
#include "src/stdio/fclose.h"
#include "src/stdio/fopen.h"
#include "src/stdio/fread.h"
@@ -15,8 +16,22 @@
#include "test/UnitTest/Test.h"
+#ifdef LIBC_TARGET_OS_IS_LINUX
+#include "hdr/fcntl_macros.h"
+#include "src/fcntl/fcntl.h"
+#include "src/stdio/fileno.h"
+#include "src/stdio/remove.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#endif
+
using LIBC_NAMESPACE::cpp::scope_exit;
+#ifdef LIBC_TARGET_OS_IS_LINUX
+using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails;
+using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
+#endif
+
TEST(LlvmLibcFOpenTest, PrintToFile) {
size_t result;
@@ -42,3 +57,83 @@ TEST(LlvmLibcFOpenTest, PrintToFile) {
ASSERT_STREQ(data, STRING);
}
}
+
+#ifdef LIBC_TARGET_OS_IS_LINUX
+class LlvmLibcFOpenModeTest
+ : public LIBC_NAMESPACE::testing::ErrnoCheckingTest {
+protected:
+ void check_exclusive_create(const char *mode) {
+ constexpr const char *TEST_FILE_NAME = "testdata/exclusive_create.test";
+ auto TEST_FILE = libc_make_test_file_path(TEST_FILE_NAME);
+ constexpr char CONTENT[] = "Preserve this content";
+
+ // Ensure the file does not exist before testing exclusive creation.
+ LIBC_NAMESPACE::remove(TEST_FILE);
+ libc_errno = 0;
+ FILE *file = LIBC_NAMESPACE::fopen(TEST_FILE, mode);
+ ASSERT_NE(file, nullptr);
+ // Keep the file until the exclusive-open and content checks are complete.
+ scope_exit remove_file(
+ [&] { EXPECT_THAT(LIBC_NAMESPACE::remove(TEST_FILE), Succeeds(0)); });
+ // Flush and close the writer before the following checks.
+ {
+ ASSERT_EQ(LIBC_NAMESPACE::fwrite(CONTENT, 1, sizeof(CONTENT) - 1, file),
+ sizeof(CONTENT) - 1);
+ EXPECT_THAT(LIBC_NAMESPACE::fclose(file), Succeeds(0));
+ }
+
+ FILE *existing = LIBC_NAMESPACE::fopen(TEST_FILE, mode);
+ EXPECT_THAT(existing, Fails(EEXIST, static_cast<void *>(nullptr)));
+ if (existing != nullptr)
+ EXPECT_THAT(LIBC_NAMESPACE::fclose(existing), Succeeds(0));
+
+ file = LIBC_NAMESPACE::fopen(TEST_FILE, "r");
+ ASSERT_NE(file, nullptr);
+ scope_exit close_file(
+ [&] { EXPECT_THAT(LIBC_NAMESPACE::fclose(file), Succeeds(0)); });
+ char buffer[sizeof(CONTENT)] = {};
+ ASSERT_EQ(LIBC_NAMESPACE::fread(buffer, 1, sizeof(buffer) - 1, file),
+ sizeof(CONTENT) - 1);
+ EXPECT_STREQ(buffer, CONTENT);
+ }
+};
+
+TEST_F(LlvmLibcFOpenModeTest, ExclusiveWrite) { check_exclusive_create("wx"); }
+
+TEST_F(LlvmLibcFOpenModeTest, ExclusiveAppend) { check_exclusive_create("ax"); }
+
+TEST_F(LlvmLibcFOpenModeTest, CloseOnExec) {
+ constexpr const char *TEST_FILE_NAME = "testdata/close_on_exec.test";
+ auto TEST_FILE = libc_make_test_file_path(TEST_FILE_NAME);
+ {
+ FILE *file = LIBC_NAMESPACE::fopen(TEST_FILE, "w");
+ ASSERT_NE(file, nullptr);
+
+ EXPECT_THAT(LIBC_NAMESPACE::fcntl(LIBC_NAMESPACE::fileno(file), F_GETFD),
+ Succeeds(0));
+ EXPECT_THAT(LIBC_NAMESPACE::fclose(file), Succeeds(0));
+ }
+ {
+ FILE *file = LIBC_NAMESPACE::fopen(TEST_FILE, "we");
+ ASSERT_NE(file, nullptr);
+
+ EXPECT_THAT(LIBC_NAMESPACE::fcntl(LIBC_NAMESPACE::fileno(file), F_GETFD),
+ Succeeds(FD_CLOEXEC));
+ EXPECT_THAT(LIBC_NAMESPACE::fclose(file), Succeeds(0));
+ }
+}
+
+TEST_F(LlvmLibcFOpenModeTest, ReadIgnoresExclusiveModifier) {
+ constexpr const char *TEST_FILE_NAME = "testdata/read_exclusive.test";
+ auto TEST_FILE = libc_make_test_file_path(TEST_FILE_NAME);
+ FILE *file = LIBC_NAMESPACE::fopen(TEST_FILE, "w");
+ ASSERT_NE(file, nullptr);
+ scope_exit remove_file(
+ [&] { EXPECT_THAT(LIBC_NAMESPACE::remove(TEST_FILE), Succeeds(0)); });
+ ASSERT_THAT(LIBC_NAMESPACE::fclose(file), Succeeds(0));
+
+ file = LIBC_NAMESPACE::fopen(TEST_FILE, "rx");
+ ASSERT_NE(file, nullptr);
+ EXPECT_THAT(LIBC_NAMESPACE::fclose(file), Succeeds(0));
+}
+#endif
diff --git a/libc/test/src/stdio/freopen_test.cpp b/libc/test/src/stdio/freopen_test.cpp
index 6f824c841b2d74..35b9685c3abbde 100644
--- a/libc/test/src/stdio/freopen_test.cpp
+++ b/libc/test/src/stdio/freopen_test.cpp
@@ -11,6 +11,7 @@
///
//===----------------------------------------------------------------------===//
+#include "src/__support/CPP/scope.h"
#include "src/fcntl/fcntl.h"
#include "src/stdio/clearerr.h"
#include "src/stdio/fclose.h"
@@ -88,6 +89,40 @@ TEST_F(LlvmLibcFreopenTest, ReopenFile) {
verify_file_content(FILENAME_B, CONTENT_B);
}
+#ifdef LIBC_TARGET_OS_IS_LINUX
+TEST_F(LlvmLibcFreopenTest, CloseOnExec) {
+ const auto FILENAME =
+ libc_make_test_file_path("freopen_close_on_exec_enabled.test");
+
+ ::FILE *file = LIBC_NAMESPACE::fopen(FILENAME, "w");
+ ASSERT_NE(file, nullptr);
+ LIBC_NAMESPACE::cpp::scope_exit close_file(
+ [&] { EXPECT_THAT(LIBC_NAMESPACE::fclose(file), Succeeds(0)); });
+ ASSERT_THAT(LIBC_NAMESPACE::fcntl(LIBC_NAMESPACE::fileno(file), F_GETFD),
+ Succeeds(0));
+
+ ASSERT_EQ(LIBC_NAMESPACE::freopen(FILENAME, "we", file), file);
+ EXPECT_THAT(LIBC_NAMESPACE::fcntl(LIBC_NAMESPACE::fileno(file), F_GETFD),
+ Succeeds(FD_CLOEXEC));
+}
+
+TEST_F(LlvmLibcFreopenTest, ClearCloseOnExec) {
+ const auto FILENAME =
+ libc_make_test_file_path("freopen_close_on_exec_cleared.test");
+
+ ::FILE *file = LIBC_NAMESPACE::fopen(FILENAME, "we");
+ ASSERT_NE(file, nullptr);
+ LIBC_NAMESPACE::cpp::scope_exit close_file(
+ [&] { EXPECT_THAT(LIBC_NAMESPACE::fclose(file), Succeeds(0)); });
+ ASSERT_THAT(LIBC_NAMESPACE::fcntl(LIBC_NAMESPACE::fileno(file), F_GETFD),
+ Succeeds(FD_CLOEXEC));
+
+ ASSERT_EQ(LIBC_NAMESPACE::freopen(FILENAME, "w", file), file);
+ EXPECT_THAT(LIBC_NAMESPACE::fcntl(LIBC_NAMESPACE::fileno(file), F_GETFD),
+ Succeeds(0));
+}
+#endif
+
TEST_F(LlvmLibcFreopenTest, NullFilenameModeChange) {
const auto FILENAME = libc_make_test_file_path("freopen_null_filename.test");
>From fc65e60bf615aee624915de907570f4fc54af4b3 Mon Sep 17 00:00:00 2001
From: Anshul Nigham <nigham at google.com>
Date: Tue, 6 Oct 2026 14:59:00 -0700
Subject: [PATCH 40/46] [Bazel] Add FlowSensitive/Models headers to
clang:analysis (#229578)
Fixes bazel build failure introduced in commits e03123a63162 and
f0d6540d5307, where GtestModelHelpers.h was added to
lib/Analysis/FlowSensitive/Models and included in Models/*.cpp.
---
utils/bazel/llvm-project-overlay/clang/BUILD.bazel | 2 ++
1 file changed, 2 insertions(+)
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index 6234256407e833..dafe982730a18f 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -1247,6 +1247,7 @@ cc_library(
name = "analysis",
srcs = glob([
"lib/Analysis/FlowSensitive/Models/*.cpp",
+ "lib/Analysis/FlowSensitive/Models/*.h",
"lib/Analysis/FlowSensitive/*.cpp",
"lib/Analysis/LifetimeSafety/*.cpp",
"lib/Analysis/LifetimeSafety/*.h",
@@ -1260,6 +1261,7 @@ cc_library(
includes = [
"include",
"lib/Analysis/FlowSensitive",
+ "lib/Analysis/FlowSensitive/Models",
],
textual_hdrs = glob([
"include/clang/Analysis/**/*.def",
>From 0c39f0df5d61b632e1ea5c748dcace9f655c872c Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak at amd.com>
Date: Tue, 6 Oct 2026 15:00:33 -0700
Subject: [PATCH 41/46] [mlir] Migrate AMDGPU/ROCDL to targets, not chipset
versions (#223563)
**migration tl;dr:** Replace usages of `amdgpu::Chipset` with
`ROCDL::TargetInfo`, ideally move from `chipset=` to `arch=`. If you
don't use upstream pipelines, call
'TargetInfo::migrateArchFeaturesToModuleFlags` at the appropriate
location.
Further note: if you've got a build pipeline that's getting a `gfxXXX`
name from something like `rocm_agent_enumerator`, using a full triple
name like the ones you get from `rocminfo` is preferred.
`amdgpu::Chipset` was an awkward hack that was hard to keep up to date
with changes in the compiler/new architectures, and didn't properly
support generic targets (and has been strongly disfavored by the
compiler team).
This PR replaces `amdgpu::Chipset` with `ROCDL::TargetInfo`, a
structure that uses LLVM's TargetParser and the underlying LLVM
features tables to get the real nature of the target being compiled
for.
This also helps MLIR move to
new-style (`-mtriple=amdgpuX.YZ-amd-amdhsa`) over "old
style" (`-mtriple=amdgcn-amd-amdhsa -mcpu=gfxXYZ`) triples.
The utility structure is moved from AMDGPU to ROCDL, both because it's
tied to LLVM rather directly and because projects like Triton should
be able to use these feature tests without pulling in the AMDGPU
dialect and its memref dependencies.
This migration also fixes a few correctness issues:
- Atomic emulation was producing floating-point additions that don't
exist on gfx90c (even though it's "after" gfx90a) and gfx908's more
precise about where emulation is needed.
- gfx90c was also being handed a bare `s_barrier`, but it has no
hardware barrier back-off, so it needs the inline asm workaround.
- gfx950 won't allow xf32 MFMAs anymore.
- permlane_swap forms that don't exist on some architectures no
longer lower.
- gfx11.7 is now listed as an OCP FP8-having target.
Some checks still need to check for a generation (ex. when encoding
s_waitcnt or what the semantics of a WMMA are) go through an
`isGeneration(N)` method, which checks for having instructions from
generation N but not N+1.
(The barrier lowering has been reordered to account for gfx13 not
having, but also not needing, BackOffBarrier.)
This migration renames the `chipset` or `chip` options on most passes to
`arch`, but keeps the old name for compatibility.
Please note that, because of changes to how LLVM handles xnack and
sramecc (they aren't target features anymore), you need to set
`rocdl.xnack` and `rocdl.sramecc` on the modules you're translating if
you know the values of those modifiers.
`TargetInfo::migrateArchFeaturesToModuleFlags(Operation *op)` will do
this for you.
Chipset is deprecated instead of being removed so that folks have time
to migrate.
Pre-commit tests in #220104.
AI disclosure: I steered this, Claude wrote the code, I tried to clean
up the docs.
---
mlir/docs/ReleaseNotes.md | 35 +
.../Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h | 4 +-
.../Conversion/ArithToAMDGPU/ArithToAMDGPU.h | 4 +-
.../Conversion/GPUToROCDL/GPUToROCDLPass.h | 7 +-
.../mlir/Conversion/MathToROCDL/MathToROCDL.h | 8 +-
mlir/include/mlir/Conversion/Passes.td | 61 +-
.../mlir/Dialect/AMDGPU/Transforms/Passes.h | 4 +-
.../mlir/Dialect/AMDGPU/Transforms/Passes.td | 21 +-
.../mlir/Dialect/AMDGPU/Utils/Chipset.h | 17 +-
.../mlir/Dialect/GPU/Pipelines/Passes.h | 29 +-
.../GPU/TransformOps/GPUTransformOps.td | 27 +-
.../mlir/Dialect/GPU/Transforms/Passes.h | 15 +-
.../mlir/Dialect/GPU/Transforms/Passes.td | 19 +-
.../AMDGPUToROCDL/AMDGPUToROCDL.cpp | 640 +++++++++---------
.../ArithToAMDGPU/ArithToAMDGPU.cpp | 51 +-
.../GPUToROCDL/LowerGpuOpsToROCDLOps.cpp | 61 +-
.../Conversion/MathToROCDL/MathToROCDL.cpp | 27 +-
.../Dialect/AMDGPU/Transforms/CMakeLists.txt | 1 +
.../AMDGPU/Transforms/EmulateAtomics.cpp | 86 +--
.../GPU/Pipelines/GPUToROCDLPipeline.cpp | 14 +-
.../GPU/TransformOps/GPUTransformOps.cpp | 45 +-
.../GPU/Transforms/PromoteShuffleToAMDGPU.cpp | 10 +-
.../GPU/Transforms/ROCDLAttachTarget.cpp | 91 ++-
.../GPU/Transforms/SubgroupReduceLowering.cpp | 34 +-
.../8-bit-floats-ocp-gfx1170.mlir | 23 -
.../AMDGPUToROCDL/8-bit-floats-ocp.mlir | 5 +-
.../AMDGPUToROCDL/8-bit-floats.mlir | 2 +-
.../AMDGPUToROCDL/amdgpu-to-rocdl.mlir | 14 +-
.../Conversion/AMDGPUToROCDL/dot-gfx11.mlir | 2 +-
.../Conversion/AMDGPUToROCDL/dot-gfx12.mlir | 2 +-
.../Conversion/AMDGPUToROCDL/dot-gfx9.mlir | 2 +-
.../Conversion/AMDGPUToROCDL/dot-invalid.mlir | 4 +-
mlir/test/Conversion/AMDGPUToROCDL/dpp.mlir | 7 +-
.../Conversion/AMDGPUToROCDL/gfx1250.mlir | 2 +-
.../AMDGPUToROCDL/global-prefetch.mlir | 2 +-
.../AMDGPUToROCDL/global_transpose_load.mlir | 6 +-
.../AMDGPUToROCDL/lds-barrier-gfx90c.mlir | 5 +-
.../AMDGPUToROCDL/load_lds-gfx950.mlir | 4 +-
.../Conversion/AMDGPUToROCDL/load_lds.mlir | 4 +-
.../AMDGPUToROCDL/memory_counter_wait.mlir | 8 +-
.../memory_counter_wait_tensor.mlir | 2 +-
.../memory_counter_wait_unsupported.mlir | 6 +-
.../AMDGPUToROCDL/mfma-fp8-invalid.mlir | 21 +
.../Conversion/AMDGPUToROCDL/mfma-gfx950.mlir | 13 +-
.../mfma-reduce-precision-invalid.mlir | 23 +
mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir | 2 +-
.../Conversion/AMDGPUToROCDL/packed-ext.mlir | 2 +-
.../AMDGPUToROCDL/packed-trunc-invalid.mlir | 2 +-
.../AMDGPUToROCDL/packed-trunc.mlir | 2 +-
.../permlane-gfx1200-invalid.mlir | 21 +
.../permlane-gfx1250-invalid.mlir | 11 +
.../AMDGPUToROCDL/permlane-gfx1250.mlir | 12 +
.../AMDGPUToROCDL/permlane-var.mlir | 2 +-
.../Conversion/AMDGPUToROCDL/permlane.mlir | 3 +-
.../AMDGPUToROCDL/sparse-mfma-gfx950.mlir | 2 +-
.../Conversion/AMDGPUToROCDL/sparse-mfma.mlir | 2 +-
.../Conversion/AMDGPUToROCDL/swizzle.mlir | 2 +-
.../AMDGPUToROCDL/swmmac-gfx12.mlir | 2 +-
.../AMDGPUToROCDL/swmmac-gfx1250.mlir | 2 +-
.../AMDGPUToROCDL/transpose_load.mlir | 4 +-
.../AMDGPUToROCDL/transpose_load_gfx1250.mlir | 2 +-
.../transpose_load_gfx1250_invalid.mlir | 2 +-
.../transpose_load_gfx950_invalid.mlir | 2 +-
.../AMDGPUToROCDL/transpose_load_reject.mlir | 2 +-
.../Conversion/AMDGPUToROCDL/wmma-gfx11.mlir | 2 +-
.../Conversion/AMDGPUToROCDL/wmma-gfx12.mlir | 2 +-
.../AMDGPUToROCDL/wmma-gfx1250.mlir | 2 +-
.../ArithToAMDGPU/16-bit-floats.mlir | 2 +-
.../8-bit-float-saturation-ocp.mlir | 4 +-
.../ArithToAMDGPU/8-bit-float-saturation.mlir | 2 +-
.../ArithToAMDGPU/8-bit-floats-ocp.mlir | 6 +-
.../ArithToAMDGPU/8-bit-floats.mlir | 2 +-
.../deprecated-chipset-alias.mlir | 27 +
.../ArithToAMDGPU/scaling-extf.mlir | 4 +-
.../ArithToAMDGPU/scaling-truncf-tensor.mlir | 2 +-
.../ArithToAMDGPU/scaling-truncf.mlir | 4 +-
.../Conversion/GPUCommon/lower-global-id.mlir | 2 +-
.../GPUCommon/lower-memory-space-attrs.mlir | 2 +-
.../GPUCommon/memory-attrbution.mlir | 2 +-
.../GPUCommon/memref-arg-attrs.mlir | 2 +-
.../GPUCommon/memref-arg-noalias-attrs.mlir | 2 +-
.../GPUCommon/memref-arg-noalias-warning.mlir | 2 +-
.../GPUToROCDL/constant-address-space.mlir | 2 +-
.../GPUToROCDL/gpu-to-rocdl-barrier.mlir | 4 +-
.../gpu-to-rocdl-barriers-gfx12.mlir | 2 +-
.../GPUToROCDL/gpu-to-rocdl-hip.mlir | 4 +-
.../gpu-to-rocdl-invalid-ballot.mlir | 2 +-
.../gpu-to-rocdl-invalid-dialect.mlir | 2 +-
.../gpu-to-rocdl-invalid-named-barrier.mlir | 2 +-
.../gpu-to-rocdl-named-barrier-non-const.mlir | 2 +-
.../GPUToROCDL/gpu-to-rocdl-opencl.mlir | 2 +-
.../GPUToROCDL/gpu-to-rocdl-subgroup-id.mlir | 4 +-
.../Conversion/GPUToROCDL/gpu-to-rocdl.mlir | 6 +-
mlir/test/Conversion/GPUToROCDL/memref.mlir | 4 +-
.../private-func-no-c-interface.mlir | 2 +-
.../Conversion/MathToROCDL/math-to-rocdl.mlir | 4 +-
.../AMDGPU/amdgpu-emulate-atomics.mlir | 73 +-
.../GPU/promote-shuffle-amdgpu-invalid.mlir | 58 ++
.../Dialect/GPU/promote-shuffle-amdgpu.mlir | 2 +-
mlir/test/Dialect/LLVMIR/attach-targets.mlir | 2 +-
.../LLVMIR/rocdl-attach-target-arch.mlir | 105 +++
.../GPU/ROCM/gpu-lower-to-rocdl-pipeline.mlir | 2 +-
.../Integration/GPU/ROCM/gpu-to-hsaco.mlir | 2 +-
mlir/test/Integration/GPU/ROCM/printf.mlir | 2 +-
.../Integration/GPU/ROCM/two-modules.mlir | 2 +-
mlir/test/Integration/GPU/ROCM/vecadd.mlir | 2 +-
.../GPU/ROCM/vector-transferops.mlir | 2 +-
mlir/test/lib/Dialect/GPU/TestGpuRewrite.cpp | 13 +-
.../Dialect/AMDGPU/AMDGPUUtilsTest.cpp | 7 +
109 files changed, 1205 insertions(+), 727 deletions(-)
delete mode 100644 mlir/test/Conversion/AMDGPUToROCDL/8-bit-floats-ocp-gfx1170.mlir
create mode 100644 mlir/test/Conversion/AMDGPUToROCDL/mfma-fp8-invalid.mlir
create mode 100644 mlir/test/Conversion/AMDGPUToROCDL/mfma-reduce-precision-invalid.mlir
create mode 100644 mlir/test/Conversion/AMDGPUToROCDL/permlane-gfx1200-invalid.mlir
create mode 100644 mlir/test/Conversion/AMDGPUToROCDL/permlane-gfx1250-invalid.mlir
create mode 100644 mlir/test/Conversion/AMDGPUToROCDL/permlane-gfx1250.mlir
create mode 100644 mlir/test/Conversion/ArithToAMDGPU/deprecated-chipset-alias.mlir
create mode 100644 mlir/test/Dialect/GPU/promote-shuffle-amdgpu-invalid.mlir
create mode 100644 mlir/test/Dialect/LLVMIR/rocdl-attach-target-arch.mlir
diff --git a/mlir/docs/ReleaseNotes.md b/mlir/docs/ReleaseNotes.md
index e24b2790d82555..24a4a5f1b6ddd4 100644
--- a/mlir/docs/ReleaseNotes.md
+++ b/mlir/docs/ReleaseNotes.md
@@ -20,6 +20,41 @@ inherent attribute and property directly or include `prop-dict`. Under the
default rules, `attr-dict` contains only discardable attributes. See
[the guide](DefiningDialects/_index.md/#strict-property-assembly-formats).
+### GPU/AMDGPU Changes
+
+- `mlir::amdgpu::Chipset` is deprecated in favour of `mlir::ROCDL::TargetInfo`,
+ which describes a target by its triple, subarch, and the resolved set of
+ target features from LLVM's own tables. Lowerings should ask whether a target
+ has a feature rather than inaccurately compare chipset versions.
+ `TargetInfo` also represents generic targets such as `gfx9-4-generic` and,
+ unlike `Chipset`, explicitly stores the wavesize for targets where it is
+ configurable.
+- The `chipset` option in AMDGPU passes is renamed to an `arch` option, which
+ uses Clang target naming syntax. It accepts a GPU name with optional
+ modifiers (`gfx942`, `gfx942:xnack+`, `gfx9-4-generic`), a triple
+ (`amdgpu9.42-amd-amdhsa`), or a full target ID
+ (`amdgpu9.0a-amd-amdhsa--gfx90a:sramecc+:xnack-`, which is what `rocminfo`
+ prints for a device's ISA). `chipset` or `chip` remain as compatibility
+ names. The default arch is `invalid`, so a target must be passed explicitly,
+ removing the old "fallback" `gfx000` GPU.
+- Wavefront size is not a target-ID feature, so `convert-gpu-to-rocdl` takes it
+ as a separate `wavesize` option (32, 64, or 0 for the architecture's
+ default). The `wave64` flag on `gpu-lower-to-rocdl-pipeline` and on
+ `rocdl-attach-target` is likewise replaced by the same `wavesize` option,
+ which has the same allowed values.
+- In keeping with broader LLVM changes, `xnack` and `sramecc` are no longer
+ architecture features but module flags. In keeping with Clang, the target
+ specifier still includes these xnack/sramecc flags where they're configurable,
+ but lowering passes now convert these to module flags. Downstream users should
+ call `migrateArchFeaturesToModuleFlags` to lower these attributes in custom
+ pipelines.
+- `rocdl-attach-target` gains `arch` alongside its existing `triple`, `chip` and
+ `features`. When `arch` is given, it overrides `triple` and `chip`, and
+ handles xnack/sramecc modifier migration.
+- `gpu-lower-to-rocdl-pipeline`, however, only supports `arch`/`chip` now, as
+ manually setting a triple/chipset/features combination is a low-level
+ operation that should require a manual `rocdl-attach-target` call.
+
## LLVM 21
### GPU/NVVM Changes
diff --git a/mlir/include/mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h b/mlir/include/mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h
index 393658652dbac4..861bbe0cc8f754 100644
--- a/mlir/include/mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h
+++ b/mlir/include/mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h
@@ -8,7 +8,7 @@
#ifndef MLIR_CONVERSION_AMDGPUTOROCDL_AMDGPUTOROCDL_H_
#define MLIR_CONVERSION_AMDGPUTOROCDL_AMDGPUTOROCDL_H_
-#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
+#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
#include <memory>
#include <string>
@@ -27,7 +27,7 @@ class Pass;
/// populateAMDGPUTypeAndAttributeConversions().
void populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter,
RewritePatternSet &patterns,
- amdgpu::Chipset chipset);
+ const ROCDL::TargetInfo &target);
namespace amdgpu {
/// Remap common GPU memory spaces (Workgroup, Private, etc) to LLVM address
diff --git a/mlir/include/mlir/Conversion/ArithToAMDGPU/ArithToAMDGPU.h b/mlir/include/mlir/Conversion/ArithToAMDGPU/ArithToAMDGPU.h
index fd144edf77452a..e6e137759a76a3 100644
--- a/mlir/include/mlir/Conversion/ArithToAMDGPU/ArithToAMDGPU.h
+++ b/mlir/include/mlir/Conversion/ArithToAMDGPU/ArithToAMDGPU.h
@@ -9,7 +9,7 @@
#ifndef MLIR_CONVERSION_ARITHTOAMDGPU_ARITHTOAMDGPU_H
#define MLIR_CONVERSION_ARITHTOAMDGPU_ARITHTOAMDGPU_H
-#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
+#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
#include "mlir/IR/PatternMatch.h"
#include <memory>
#include <string>
@@ -31,7 +31,7 @@ namespace arith {
void populateArithToAMDGPUConversionPatterns(
RewritePatternSet &patterns, bool convertFP8Arithmetic,
bool saturateFP8Truncf, bool allowPackedF16Rtz, bool supportsScaledExtTrunc,
- amdgpu::Chipset chipset, PatternBenefit benefit = 1);
+ const ROCDL::TargetInfo &target, PatternBenefit benefit = 1);
} // namespace arith
} // namespace mlir
diff --git a/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h b/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h
index 220da0ad3c08fe..494ee8cfa11abd 100644
--- a/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h
+++ b/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h
@@ -10,6 +10,7 @@
#include "mlir/Conversion/GPUToROCDL/Runtimes.h"
#include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
+#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
#include <memory>
namespace mlir {
@@ -21,10 +22,6 @@ class RewritePatternSet;
template <typename OpT>
class OperationPass;
-namespace amdgpu {
-struct Chipset;
-} // namespace amdgpu
-
namespace gpu {
class GPUModuleOp;
} // namespace gpu
@@ -38,7 +35,7 @@ class GPUModuleOp;
void populateGpuToROCDLConversionPatterns(const LLVMTypeConverter &converter,
RewritePatternSet &patterns,
gpu::amd::Runtime runtime,
- amdgpu::Chipset chipset);
+ const ROCDL::TargetInfo &target);
/// Configure target to convert from the GPU dialect to ROCDL.
void configureGpuToROCDLConversionLegality(ConversionTarget &target);
diff --git a/mlir/include/mlir/Conversion/MathToROCDL/MathToROCDL.h b/mlir/include/mlir/Conversion/MathToROCDL/MathToROCDL.h
index 60f1888569362b..8ba104972abffe 100644
--- a/mlir/include/mlir/Conversion/MathToROCDL/MathToROCDL.h
+++ b/mlir/include/mlir/Conversion/MathToROCDL/MathToROCDL.h
@@ -9,7 +9,7 @@
#define MLIR_CONVERSION_MATHTOROCDL_MATHTOROCDL_H_
#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
-#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
+#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
#include "mlir/IR/PatternMatch.h"
#include <memory>
@@ -20,11 +20,11 @@ class Pass;
#include "mlir/Conversion/Passes.h.inc"
/// Populate the given list with patterns that convert from Math to ROCDL calls.
-// `chipset` specifies the AMDGPU chipset to target. If `std::nullopt`,
-// none of the chipset dependent patterns are added.
+// `target` describes the AMDGPU target. If `std::nullopt`, none of the
+// target-dependent patterns are added.
void populateMathToROCDLConversionPatterns(
const LLVMTypeConverter &converter, RewritePatternSet &patterns,
- std::optional<amdgpu::Chipset> chipset);
+ std::optional<ROCDL::TargetInfo> target);
} // namespace mlir
#endif // MLIR_CONVERSION_MATHTOROCDL_MATHTOROCDL_H_
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index f7a3954547cf75..99ccd9df246bcf 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -130,9 +130,18 @@ def ConvertAMDGPUToROCDLPass : Pass<"convert-amdgpu-to-rocdl"> {
"LLVM::LLVMDialect",
"ROCDL::ROCDLDialect",
];
- let options = [Option<"chipset", "chipset", "std::string",
- /*default=*/"\"gfx000\"",
- "Chipset that these operations will run on">];
+ let options = [
+ Option<"arch", "arch", "std::string",
+ /*default=*/"\"invalid\"",
+ "Target architecture, as in Clang, with optional target-ID "
+ "modifiers. New-style triples such as amdgpu9.42-amd-amdhsa are "
+ "preferred, and can be extended to a full target ID like "
+ "amdgpu9.42-amd-amdhsa--gfx942:xnack+:sramecc-. Bare chip names "
+ "like gfx1250 or gfx942:xnack- are also supported. Defaults to an "
+ "invalid target so that one must be given explicitly">,
+ Option<"chipset", "chipset", "std::string", /*default=*/"\"\"",
+ "Deprecated alias for 'arch'.">,
+ ];
}
//===----------------------------------------------------------------------===//
@@ -150,9 +159,16 @@ def ArithToAMDGPUConversionPass : Pass<"convert-arith-to-amdgpu"> {
let dependentDialects = ["amdgpu::AMDGPUDialect", "vector::VectorDialect"];
let options = [
- Option<"chipset", "chipset", "std::string",
- /*default=*/"\"gfx000\"",
- "Chipset that these operations will run on">,
+ Option<"arch", "arch", "std::string",
+ /*default=*/"\"invalid\"",
+ "Target architecture, as in Clang, with optional target-ID "
+ "modifiers. New-style triples such as amdgpu9.42-amd-amdhsa are "
+ "preferred, and can be extended to a full target ID like "
+ "amdgpu9.42-amd-amdhsa--gfx942:xnack+:sramecc-. Bare chip names "
+ "like gfx1250 or gfx942:xnack- are also supported. Defaults to an "
+ "invalid target so that one must be given explicitly">,
+ Option<"chipset", "chipset", "std::string", /*default=*/"\"\"",
+ "Deprecated alias for 'arch'.">,
Option<"saturateFP8Truncf", "saturate-fp8-truncf", "bool",
/*default=*/"false",
"Use saturating truncation for 8-bit float types">,
@@ -684,9 +700,19 @@ def ConvertGpuOpsToROCDLOps : Pass<"convert-gpu-to-rocdl", "gpu::GPUModuleOp"> {
"memref::MemRefDialect",
];
let options = [
- Option<"chipset", "chipset", "std::string",
- /*default=*/"\"gfx000\"",
- "Chipset that these operations will run on">,
+ Option<"arch", "arch", "std::string",
+ /*default=*/"\"invalid\"",
+ "Target architecture, as in Clang, with optional target-ID "
+ "modifiers. New-style triples such as amdgpu9.42-amd-amdhsa are "
+ "preferred, and can be extended to a full target ID like "
+ "amdgpu9.42-amd-amdhsa--gfx942:xnack+:sramecc-. Bare chip names "
+ "like gfx1250 or gfx942:xnack- are also supported. Defaults to an "
+ "invalid target so that one must be given explicitly">,
+ Option<"chipset", "chipset", "std::string", /*default=*/"\"\"",
+ "Deprecated alias for 'arch'.">,
+ Option<"waveSize", "wavesize", "unsigned", /*default=*/"0",
+ "Wavefront size (32 or 64) for targets that run at either, or 0 to "
+ "use the architecture's default">,
Option<"indexBitwidth", "index-bitwidth", "unsigned",
/*default=kDeriveIndexBitwidthFromDataLayout*/ "0",
"Bitwidth of the index type, 0 to use size of machine word">,
@@ -851,9 +877,9 @@ def ConvertMathToROCDL : Pass<"convert-math-to-rocdl", "ModuleOp"> {
let description = [{
This pass converts supported Math ops to ROCDL library calls.
- The chipset option specifies the target AMDGPU architecture. If the chipset
- is empty, none of the chipset-dependent patterns are added, and the pass
- will not attempt to parse the chipset.
+ The `arch` option specifies the target AMDGPU architecture. If it is empty,
+ none of the target-dependent patterns are added and the pass does not
+ resolve a target.
}];
let dependentDialects = [
"arith::ArithDialect",
@@ -861,9 +887,14 @@ def ConvertMathToROCDL : Pass<"convert-math-to-rocdl", "ModuleOp"> {
"ROCDL::ROCDLDialect",
"vector::VectorDialect",
];
- let options = [Option<"chipset", "chipset", "std::string",
- /*default=*/"\"\"",
- "Chipset that these operations will run on">];
+ let options = [
+ Option<"arch", "arch", "std::string", /*default=*/"\"\"",
+ "Target architecture, as in Clang, with optional target-ID "
+ "modifiers (e.g. amdgpu9.42-amd-amdhsa, gfx942, gfx942:xnack-). "
+ "If empty, no target-dependent patterns are added">,
+ Option<"chipset", "chipset", "std::string", /*default=*/"\"\"",
+ "Deprecated alias for 'arch'.">,
+ ];
}
//===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
index 48e7658568f86f..c8ec368231f835 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
@@ -13,7 +13,7 @@
#ifndef MLIR_DIALECT_AMDGPU_TRANSFORMS_PASSES_H_
#define MLIR_DIALECT_AMDGPU_TRANSFORMS_PASSES_H_
-#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
+#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Pass/Pass.h"
@@ -29,7 +29,7 @@ namespace amdgpu {
void populateAmdgpuEmulateAtomicsPatterns(ConversionTarget &target,
RewritePatternSet &patterns,
- Chipset chipset,
+ const ROCDL::TargetInfo &targetInfo,
PatternBenefit benefit = 1);
void populateAmdgpuResolveStridedMetadataPatterns(RewritePatternSet &patterns,
diff --git a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
index 7dd7ac750a9eb2..a9eef54a0776e4 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
@@ -16,19 +16,28 @@
include "mlir/Pass/PassBase.td"
def AmdgpuEmulateAtomicsPass : Pass<"amdgpu-emulate-atomics"> {
- let summary = "Emulate atomic operations on chipsets that do not support them";
+ let summary = "Emulate atomic operations the target does not support";
let description = [{
- This pass rewrites any AMDGPU-specific atomic operation that is not supported
- on the given `chipset` into a compare-and-swap loop.
+ This pass rewrites any AMDGPU-specific atomic operation that the target does
+ not support into a compare-and-swap loop.
}];
let dependentDialects = [
"cf::ControlFlowDialect",
"arith::ArithDialect",
"vector::VectorDialect"
];
- let options = [Option<"chipset", "chipset", "std::string",
- /*default=*/"\"gfx000\"",
- "Chipset that these operations will run on">];
+ let options = [
+ Option<"arch", "arch", "std::string",
+ /*default=*/"\"invalid\"",
+ "Target architecture, as in Clang, with optional target-ID "
+ "modifiers. New-style triples such as amdgpu9.42-amd-amdhsa are "
+ "preferred, and can be extended to a full target ID like "
+ "amdgpu9.42-amd-amdhsa--gfx942:xnack+:sramecc-. Bare chip names "
+ "like gfx1250 or gfx942:xnack- are also supported. Defaults to an "
+ "invalid target so that one must be given explicitly">,
+ Option<"chipset", "chipset", "std::string", /*default=*/"\"\"",
+ "Deprecated alias for 'arch'.">,
+ ];
}
def AmdgpuResolveStridedMetadataPass : Pass<"amdgpu-resolve-strided-metadata"> {
diff --git a/mlir/include/mlir/Dialect/AMDGPU/Utils/Chipset.h b/mlir/include/mlir/Dialect/AMDGPU/Utils/Chipset.h
index 256065537aeb9b..f5d1b73909c441 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/Utils/Chipset.h
+++ b/mlir/include/mlir/Dialect/AMDGPU/Utils/Chipset.h
@@ -9,16 +9,20 @@
#define MLIR_DIALECT_AMDGPU_UTILS_CHIPSET_H_
#include "mlir/Support/LLVM.h"
+#include "llvm/Support/Compiler.h"
#include <tuple>
namespace mlir::amdgpu {
-
/// Represents the amdgpu gfx chipset version, e.g., gfx90a, gfx942, gfx1103.
/// Note that the leading digits form a decimal number, while the last two
/// digits form a hexadecimal number. For example:
/// gfx942 --> major = 9, minor = 0x4, stepping = 0x2
/// gfx90a --> major = 9, minor = 0x0, stepping = 0xa
/// gfx1103 --> major = 11, minor = 0x0, stepping = 0x3
+///
+/// \deprecated Use `mlir::ROCDL::TargetInfo` instead, and rely on target
+/// features rather than about version numbers. Will be removed after one
+/// release.
struct Chipset {
unsigned majorVersion = 0; // The major version (decimal).
unsigned minorVersion = 0; // The minor version (hexadecimal).
@@ -30,6 +34,10 @@ struct Chipset {
/// Parses the chipset version string and returns the chipset on success, and
/// failure otherwise.
+ ///
+ /// \deprecated Use `ROCDL::TargetInfo::get`.
+ LLVM_DEPRECATED_WITH_FIXIT("Chipset struct deprecated",
+ "use ROCDL::TargetInfo::get instead")
static FailureOr<Chipset> parse(StringRef name);
std::tuple<unsigned, unsigned, unsigned> asTuple() const {
@@ -49,11 +57,16 @@ struct Chipset {
#undef DEFINE_COMP_OPERATOR
};
+/// \deprecated Test `llvm::AMDGPU::FEAT_OCP_FP8_CONVERSION_INSTS` on a
+/// `ROCDL::TargetInfo` instead. This misses gfx11.7, which does have the OCP
+/// fp8 conversions.
+LLVM_DEPRECATED_WITH_FIXIT(
+ "Chipset is deperecated",
+ "test FEAT_OCP_FP8_CONVERSION_INSTS on a ROCDL::TargetInfo")
inline bool hasOcpFp8(const Chipset &chipset) {
return (chipset.majorVersion == 9 && chipset.minorVersion >= 5) ||
chipset.majorVersion >= 12;
}
-
} // namespace mlir::amdgpu
#endif
diff --git a/mlir/include/mlir/Dialect/GPU/Pipelines/Passes.h b/mlir/include/mlir/Dialect/GPU/Pipelines/Passes.h
index 0d35c613911436..a541864e50df08 100644
--- a/mlir/include/mlir/Dialect/GPU/Pipelines/Passes.h
+++ b/mlir/include/mlir/Dialect/GPU/Pipelines/Passes.h
@@ -72,17 +72,16 @@ struct GPUToROCDLPipelineOptions
llvm::cl::desc("Bitwidth of the index type for the host (warning this "
"should be 64 until the GPU layering is fixed)"),
llvm::cl::init(64)};
- PassOptions::Option<std::string> triple{
- *this, "triple",
- llvm::cl::desc("AMDGPU target triple (e.g. amdgcn-amd-amdhsa)."),
- llvm::cl::init("amdgcn-amd-amdhsa")};
- PassOptions::Option<std::string> chip{
- *this, "chip",
+ PassOptions::Option<std::string> arch{
+ *this, "arch",
llvm::cl::desc(
- "AMDGPU target chip (e.g. gfx90a, gfx942, gfx1100). Required: "
- "AMDGCN binaries are not forward-compatible across chip families.")};
- PassOptions::Option<std::string> features{
- *this, "features", llvm::cl::desc("AMDGPU target features."),
+ "AMDGPU target architecture, as in Clang, with optional target-ID "
+ "modifiers (e.g. gfx942, gfx90a:xnack+, amdgpu9.42-amd-amdhsa, "
+ "amdgpu9.0a-amd-amdhsa--gfx90a:xnack-). Supersedes the separate "
+ "triple option this pipeline used to take. Required: AMDGCN "
+ "binaries are not forward-compatible across chip families.")};
+ PassOptions::Option<std::string> chip{
+ *this, "chip", llvm::cl::desc("Deprecated alias for 'arch'."),
llvm::cl::init("")};
PassOptions::Option<std::string> binaryFormat{
*this, "binary-format",
@@ -93,11 +92,11 @@ struct GPUToROCDLPipelineOptions
*this, "abi",
llvm::cl::desc("AMDHSA ABI version (e.g. \"500\", \"600\")."),
llvm::cl::init("600")};
- PassOptions::Option<bool> wave64{
- *this, "wave64",
- llvm::cl::desc("Use Wave64 mode (default true; wave32 if false, "
- "appropriate for RDNA / gfx10+ where supported)."),
- llvm::cl::init(true)};
+ PassOptions::Option<unsigned> waveSize{
+ *this, "wavesize",
+ llvm::cl::desc("Wavefront size (32 or 64) for targets that run at "
+ "either, or 0 to use the architecture's default."),
+ llvm::cl::init(0)};
PassOptions::Option<int> optLevel{
*this, "opt-level",
llvm::cl::desc("Optimization level for ROCDL/AMDGPU compilation."),
diff --git a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td
index 030506bba10768..d56cc1f13afd2e 100644
--- a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td
+++ b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td
@@ -61,11 +61,18 @@ def ApplyGPUToROCDLConversionPatternsOp : Op<Transform_Dialect,
let description = [{
Collects patterns that convert GPU dialect ops to ROCDL dialect ops. These
patterns require an "LLVMTypeConverter".
+
+ `arch` names the target the way Clang does: a GPU name with optional
+ target-ID modifiers ("gfx942", "gfx942:xnack+", "gfx9-4-generic"), a triple
+ ("amdgpu9.42-amd-amdhsa"), or a full target ID
+ ("amdgpu9.0a-amd-amdhsa--gfx90a:sramecc+:xnack-").
+
+ `wavesize` pins the wavefront size on targets that run at either; omitting
+ it uses the architecture's own default.
}];
- let arguments = (ins StrAttr:$chipset);
- let assemblyFormat = [{
- `chipset` `=` $chipset attr-dict
- }];
+ let arguments = (ins StrAttr:$arch,
+ OptionalAttr<I32Attr>:$wavesize);
+ let assemblyFormat = "prop-dict attr-dict";
}
//===----------------------------------------------------------------------===//
@@ -328,11 +335,15 @@ def ApplyGPUPromoteShuffleToAMDGPUPatternsOp : Op<Transform_Dialect,
let description = [{
Collects patterns that are tryin to promote `gpu.shuffle`s to specialized
AMDGPU intrinsics.
+
+ `arch` names the target the way Clang does: a GPU name with optional
+ target-ID modifiers ("gfx950", "gfx950:xnack+"), a triple
+ ("amdgpu9.50-amd-amdhsa"), or a full target ID. Omitting it collects only
+ the patterns that hold on every target.
}];
- let arguments = (ins OptionalAttr<StrAttr>:$chipset);
- let assemblyFormat = [{
- (`chipset` `=` $chipset^)? attr-dict
- }];
+ let arguments = (ins OptionalAttr<StrAttr>:$arch);
+ let assemblyFormat = "prop-dict attr-dict";
+ let hasVerifier = 1;
}
diff --git a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h
index d5c253d6c9c08c..6657ff819dea5c 100644
--- a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h
@@ -13,9 +13,9 @@
#ifndef MLIR_DIALECT_GPU_TRANSFORMS_PASSES_H_
#define MLIR_DIALECT_GPU_TRANSFORMS_PASSES_H_
-#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
#include "mlir/Dialect/GPU/IR/GPUDialect.h"
#include "mlir/Dialect/GPU/Utils/GPUUtils.h"
+#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Pass/Pass.h"
#include <optional>
@@ -76,16 +76,15 @@ void populateGpuLowerClusteredSubgroupReduceToShufflePatterns(
/// Collect a set of patterns to lower `gpu.subgroup_reduce` into `amdgpu.dpp`
/// ops over scalar types. Assumes that the subgroup has
/// `subgroupSize` lanes. Applicable only to AMD GPUs.
-void populateGpuLowerSubgroupReduceToDPPPatterns(RewritePatternSet &patterns,
- unsigned subgroupSize,
- amdgpu::Chipset chipset,
- PatternBenefit benefit = 1);
+void populateGpuLowerSubgroupReduceToDPPPatterns(
+ RewritePatternSet &patterns, unsigned subgroupSize,
+ const ROCDL::TargetInfo &target, PatternBenefit benefit = 1);
/// Disjoint counterpart of `populateGpuLowerSubgroupReduceToDPPPatterns`
/// that only matches `gpu.subgroup_reduce` ops with a `cluster_size`.
void populateGpuLowerClusteredSubgroupReduceToDPPPatterns(
- RewritePatternSet &patterns, unsigned subgroupSize, amdgpu::Chipset chipset,
- PatternBenefit benefit = 1);
+ RewritePatternSet &patterns, unsigned subgroupSize,
+ const ROCDL::TargetInfo &target, PatternBenefit benefit = 1);
/// Collect all patterns to rewrite ops within the GPU dialect.
inline void populateGpuRewritePatterns(RewritePatternSet &patterns) {
@@ -115,7 +114,7 @@ void populateGpuEliminateBarriersPatterns(RewritePatternSet &patterns);
/// Tries to promote `gpu.shuffle`s to specialized AMDGPU intrinsics.
void populateGpuPromoteShuffleToAMDGPUPatterns(
- RewritePatternSet &patterns, std::optional<amdgpu::Chipset> maybeChipset);
+ RewritePatternSet &patterns, std::optional<ROCDL::TargetInfo> target);
/// Generate the code for registering passes.
#define GEN_PASS_REGISTRATION
diff --git a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.td b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.td
index 1aaab1ee915235..090bfdba3ce8f5 100644
--- a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.td
@@ -191,12 +191,20 @@ def GpuROCDLAttachTarget: Pass<"rocdl-attach-target", ""> {
Option<"moduleMatcher", "module", "std::string",
/*default=*/ [{""}],
"Regex used to identify the modules to attach the target to.">,
+ Option<"arch", "arch", "std::string",
+ /*default=*/"\"\"",
+ "Target architecture, in the spelling the conversion passes take "
+ "(e.g. gfx942, gfx90a:xnack+, amdgpu9.4-amd-amdhsa, "
+ "amdgcn-amd-amdhsa--gfx90a:xnack-). "
+ "When non-empty this supersedes 'triple' and 'chip', and its "
+ "target-ID modifiers become the module's 'rocdl.xnack' and "
+ "'rocdl.sramecc' attributes rather than target features.">,
Option<"triple", "triple", "std::string",
/*default=*/ "\"amdgcn-amd-amdhsa\"",
- "Target triple.">,
+ "Target triple. Ignored when 'arch' is given.">,
Option<"chip", "chip", "std::string",
/*default=*/"\"gfx900\"",
- "Target chip.">,
+ "Target chip. Ignored when 'arch' is given.">,
Option<"features", "features", "std::string",
/*default=*/"\"\"",
"Target features.">,
@@ -206,9 +214,10 @@ def GpuROCDLAttachTarget: Pass<"rocdl-attach-target", ""> {
Option<"optLevel", "O", "unsigned",
/*default=*/"2",
"Optimization level.">,
- Option<"wave64Flag", "wave64", "bool",
- /*default=*/"true",
- "Use Wave64 mode.">,
+ Option<"waveSize", "wavesize", "unsigned",
+ /*default=*/"0",
+ "Wavefront size (32 or 64). 0 takes the size from the target when "
+ "'arch' is given, and Wave64 otherwise.">,
Option<"fastFlag", "fast", "bool",
/*default=*/"false",
"Enable fast relaxed math opt.">,
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 6bbe65731964f1..1bfb9616d7eefd 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -14,7 +14,6 @@
#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
#include "mlir/Dialect/AMDGPU/IR/AMDGPUEnums.h"
-#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
@@ -32,6 +31,7 @@
#include "llvm/Support/AMDGPUAddrSpace.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MathExtras.h"
#include <cstdint>
#include <optional>
@@ -43,72 +43,6 @@ namespace mlir {
using namespace mlir;
using namespace mlir::amdgpu;
-// Define commonly used chipsets versions for convenience.
-constexpr Chipset kGfx908 = Chipset(9, 0, 8);
-constexpr Chipset kGfx90a = Chipset(9, 0, 0xa);
-constexpr Chipset kGfx942 = Chipset(9, 4, 2);
-constexpr Chipset kGfx950 = Chipset(9, 5, 0);
-constexpr Chipset kGfx1200 = Chipset(12, 0, 0);
-constexpr Chipset kGfx1250 = Chipset(12, 5, 0);
-
-// Predicates mirroring the LLVM AMDGPU `HasDot{N}Insts` features that gate
-// the `v_dot*` instructions consumed by the `amdgpu.dot` lowering.
-static bool hasDot1Insts(const Chipset &chipset) {
- if (chipset.majorVersion == 9)
- return chipset >= Chipset(9, 0, 6);
- if (chipset.majorVersion == 10) {
- if (chipset.minorVersion == 1)
- return chipset.steppingVersion == 1u || chipset.steppingVersion == 2u;
- return chipset.minorVersion >= 3u;
- }
- return false;
-}
-
-static bool hasDot2Insts(const Chipset &chipset) {
- return hasDot1Insts(chipset);
-}
-
-static bool hasDot7Insts(const Chipset &chipset) {
- return chipset.majorVersion >= 11 || hasDot1Insts(chipset);
-}
-
-static bool hasDot8Insts(const Chipset &chipset) {
- return chipset.majorVersion >= 11;
-}
-
-static bool hasDot9Insts(const Chipset &chipset) {
- if (chipset.majorVersion == 11)
- return true;
- return chipset.majorVersion == 12 && chipset.minorVersion == 0;
-}
-
-static bool hasDot10Insts(const Chipset &chipset) {
- if (chipset.majorVersion == 11)
- return true;
- if (chipset.majorVersion == 12)
- return chipset.minorVersion == 0;
- return hasDot1Insts(chipset);
-}
-
-static bool hasDot11Insts(const Chipset &chipset) {
- if (chipset.majorVersion == 11)
- return chipset.minorVersion == 7u;
- return chipset.majorVersion == 12 && chipset.minorVersion == 0;
-}
-
-static bool hasDot12Insts(const Chipset &chipset) {
- if (chipset == Chipset(9, 5, 0))
- return true;
- if (chipset.majorVersion == 11)
- return true;
- return chipset.majorVersion == 12 && chipset.minorVersion == 0;
-}
-
-static bool has45BitNumRecordsBufferResource(const Chipset &chipset) {
- return chipset.majorVersion > 12 ||
- (chipset.majorVersion == 12 && chipset.minorVersion >= 5);
-}
-
/// Zero-extend or truncate the unsigned number `val` to `width` bits.
static Value convertUnsignedToInt(ConversionPatternRewriter &rewriter,
Location loc, Value val, unsigned width) {
@@ -172,11 +106,9 @@ static Value getNumRecords(ConversionPatternRewriter &rewriter, Location loc,
MemRefType memrefType,
MemRefDescriptor &memrefDescriptor,
ArrayRef<int64_t> strides, int64_t elementByteWidth,
- amdgpu::Chipset chipset, bool boundsCheck) {
- if (has45BitNumRecordsBufferResource(chipset) && !boundsCheck) {
- constexpr int64_t first45bits = (1ll << 45) - 1;
- return createI64Constant(rewriter, loc, first45bits);
- }
+ unsigned numRecordsWidth, bool boundsCheck) {
+ if (numRecordsWidth > 32 && !boundsCheck)
+ return createI64Constant(rewriter, loc, llvm::maxUIntN(numRecordsWidth));
if (memrefType.hasStaticShape() &&
!llvm::any_of(strides, ShapedType::isDynamic)) {
int64_t size = memrefType.getRank() == 0 ? 1 : 0;
@@ -202,7 +134,8 @@ static Value getNumRecords(ConversionPatternRewriter &rewriter, Location loc,
static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
Value basePointer, Value numRecords,
- bool boundsCheck, amdgpu::Chipset chipset,
+ bool boundsCheck, const ROCDL::TargetInfo &target,
+ unsigned numRecordsWidth,
Value cacheSwizzleStride = nullptr,
unsigned addressSpace = 8) {
// The stride value is generally 0. However, on MI-300 and onward, you can
@@ -210,7 +143,7 @@ static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
// and setting that stride to a cache stride.
Type i16 = rewriter.getI16Type();
Value stride;
- if (chipset.majorVersion == 9 && chipset >= kGfx942 && cacheSwizzleStride) {
+ if (target.has(llvm::AMDGPU::FEAT_GFX940_INSTS) && cacheSwizzleStride) {
Value cacheStrideZext =
LLVM::ZExtOp::create(rewriter, loc, i16, cacheSwizzleStride);
Value swizzleBit = LLVM::ConstantOp::create(
@@ -223,7 +156,7 @@ static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
}
uint32_t flags = 0;
- if (chipset >= kGfx1250) {
+ if (target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS)) {
// Flag word:
// bit 0: swizzle
// bit 1: 0 means (total_offset + payload > numRecords)
@@ -249,16 +182,14 @@ static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
// none, 3 = either swizzles or testing against offset field) RDNA only
// bits 30-31: Type (must be 0)
flags |= (7 << 12) | (4 << 15);
- if (chipset.majorVersion >= 10) {
+ if (target.has(llvm::AMDGPU::FEAT_GFX10_INSTS)) {
flags |= (1 << 24);
uint32_t oob = boundsCheck ? 3 : 2;
flags |= (oob << 28);
}
}
Value flagsConst = createI32Constant(rewriter, loc, flags);
- numRecords =
- convertUnsignedToInt(rewriter, loc, numRecords,
- has45BitNumRecordsBufferResource(chipset) ? 45 : 32);
+ numRecords = convertUnsignedToInt(rewriter, loc, numRecords, numRecordsWidth);
Type rsrcType =
LLVM::LLVMPointerType::get(rewriter.getContext(), addressSpace);
Value resource = rewriter.createOrFold<ROCDL::MakeBufferRsrcOp>(
@@ -269,11 +200,11 @@ static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
namespace {
struct FatRawBufferCastLowering
: public ConvertOpToLLVMPattern<FatRawBufferCastOp> {
- FatRawBufferCastLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<FatRawBufferCastOp>(converter),
- chipset(chipset) {}
+ FatRawBufferCastLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<FatRawBufferCastOp>(converter), target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(FatRawBufferCastOp op, FatRawBufferCastOpAdaptor adaptor,
@@ -293,11 +224,17 @@ struct FatRawBufferCastLowering
if (failed(memrefType.getStridesAndOffset(strideVals, unusedOffset)))
return op.emitOpError("Can't lower non-stride-offset memrefs");
+ std::optional<unsigned> numRecordsWidth =
+ target.getBufferResourceNumRecordsWidth();
+ if (!numRecordsWidth)
+ return op.emitOpError(
+ "buffer resource num_records width is unknown for this target");
+
Value numRecords = adaptor.getValidBytes();
if (!numRecords)
- numRecords =
- getNumRecords(rewriter, loc, memrefType, descriptor, strideVals,
- elementByteWidth, chipset, adaptor.getBoundsCheck());
+ numRecords = getNumRecords(rewriter, loc, memrefType, descriptor,
+ strideVals, elementByteWidth, *numRecordsWidth,
+ adaptor.getBoundsCheck());
Value basePointer =
adaptor.getResetOffset()
@@ -324,7 +261,8 @@ struct FatRawBufferCastLowering
Value fatPtr = makeBufferRsrc(
rewriter, loc, basePointer, numRecords, adaptor.getBoundsCheck(),
- chipset, adaptor.getCacheSwizzleStride(), /*addressSpace=*/7);
+ target, *numRecordsWidth, adaptor.getCacheSwizzleStride(),
+ /*addressSpace=*/7);
Value result = MemRefDescriptor::poison(
rewriter, loc,
@@ -349,10 +287,11 @@ struct FatRawBufferCastLowering
/// Define lowering patterns for raw buffer ops
template <typename GpuOp, typename Intrinsic>
struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
- RawBufferOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<GpuOp>(converter), chipset(chipset) {}
+ RawBufferOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<GpuOp>(converter), target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
static constexpr uint32_t maxVectorOpWidth = 128;
LogicalResult
@@ -363,7 +302,7 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
Value unconvertedMemref = gpuOp.getMemref();
MemRefType memrefType = cast<MemRefType>(unconvertedMemref.getType());
- if (chipset.majorVersion < 9)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX9_INSTS))
return gpuOp.emitOpError("raw buffer ops require GCN or higher");
Value storeData = adaptor.getODSOperands(0)[0];
@@ -468,11 +407,18 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
Value ptr = memrefDescriptor.bufferPtr(
rewriter, loc, *this->getTypeConverter(), memrefType);
- Value numRecords =
- getNumRecords(rewriter, loc, memrefType, memrefDescriptor, strides,
- elementByteWidth, chipset, adaptor.getBoundsCheck());
- Value resource = makeBufferRsrc(rewriter, loc, ptr, numRecords,
- adaptor.getBoundsCheck(), chipset);
+ std::optional<unsigned> numRecordsWidth =
+ target.getBufferResourceNumRecordsWidth();
+ if (!numRecordsWidth)
+ return gpuOp.emitOpError(
+ "buffer resource num_records width is unknown for this target");
+
+ Value numRecords = getNumRecords(
+ rewriter, loc, memrefType, memrefDescriptor, strides, elementByteWidth,
+ *numRecordsWidth, adaptor.getBoundsCheck());
+ Value resource =
+ makeBufferRsrc(rewriter, loc, ptr, numRecords, adaptor.getBoundsCheck(),
+ target, *numRecordsWidth);
args.push_back(resource);
// Indexing (voffset)
@@ -526,15 +472,18 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
/// Lgkmcnt = Waitcnt[11:8] (pre-gfx10)
/// Lgkmcnt = Waitcnt[13:8] (gfx10)
/// Lgkmcnt = Waitcnt[9:4] (gfx11)
-static FailureOr<unsigned> encodeWaitcnt(Chipset chipset, unsigned vmcnt,
- unsigned expcnt, unsigned lgkmcnt) {
- if (chipset.majorVersion < 9) {
+static FailureOr<unsigned> encodeWaitcnt(const ROCDL::TargetInfo &target,
+ unsigned vmcnt, unsigned expcnt,
+ unsigned lgkmcnt) {
+ if (target.isUnknown())
+ return failure();
+ if (!target.has(llvm::AMDGPU::FEAT_GFX9_INSTS)) {
vmcnt = std::min(15u, vmcnt);
expcnt = std::min(7u, expcnt);
lgkmcnt = std::min(15u, lgkmcnt);
return vmcnt | (expcnt << 4) | (lgkmcnt << 8);
}
- if (chipset.majorVersion == 9) {
+ if (target.isGeneration(9)) {
vmcnt = std::min(63u, vmcnt);
expcnt = std::min(7u, expcnt);
lgkmcnt = std::min(15u, lgkmcnt);
@@ -543,7 +492,7 @@ static FailureOr<unsigned> encodeWaitcnt(Chipset chipset, unsigned vmcnt,
unsigned otherCnts = (expcnt << 4) | (lgkmcnt << 8);
return lowBits | highBits | otherCnts;
}
- if (chipset.majorVersion == 10) {
+ if (target.isGeneration(10)) {
vmcnt = std::min(63u, vmcnt);
expcnt = std::min(7u, expcnt);
lgkmcnt = std::min(63u, lgkmcnt);
@@ -552,7 +501,7 @@ static FailureOr<unsigned> encodeWaitcnt(Chipset chipset, unsigned vmcnt,
unsigned otherCnts = (expcnt << 4) | (lgkmcnt << 8);
return lowBits | highBits | otherCnts;
}
- if (chipset.majorVersion == 11) {
+ if (target.isGeneration(11)) {
vmcnt = std::min(63u, vmcnt);
expcnt = std::min(7u, expcnt);
lgkmcnt = std::min(63u, lgkmcnt);
@@ -564,16 +513,16 @@ static FailureOr<unsigned> encodeWaitcnt(Chipset chipset, unsigned vmcnt,
struct MemoryCounterWaitOpLowering
: public ConvertOpToLLVMPattern<MemoryCounterWaitOp> {
MemoryCounterWaitOpLowering(const LLVMTypeConverter &converter,
- Chipset chipset)
- : ConvertOpToLLVMPattern<MemoryCounterWaitOp>(converter),
- chipset(chipset) {}
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<MemoryCounterWaitOp>(converter), target(target) {
+ }
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(MemoryCounterWaitOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset.majorVersion >= 12) {
+ if (target.has(llvm::AMDGPU::FEAT_GFX12_INSTS)) {
Location loc = op.getLoc();
if (std::optional<int> ds = adaptor.getDs())
ROCDL::WaitDscntOp::create(rewriter, loc, *ds);
@@ -618,7 +567,7 @@ struct MemoryCounterWaitOpLowering
vmcnt = getVal(store);
}
- FailureOr<unsigned> waitcnt = encodeWaitcnt(chipset, vmcnt, exp, ds);
+ FailureOr<unsigned> waitcnt = encodeWaitcnt(target, vmcnt, exp, ds);
if (failed(waitcnt))
return op.emitOpError("unsupported chipset");
@@ -628,18 +577,24 @@ struct MemoryCounterWaitOpLowering
};
struct LDSBarrierOpLowering : public ConvertOpToLLVMPattern<LDSBarrierOp> {
- LDSBarrierOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<LDSBarrierOp>(converter), chipset(chipset) {}
+ LDSBarrierOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<LDSBarrierOp>(converter), target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(LDSBarrierOp op, LDSBarrierOp::Adaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
Location loc = op.getLoc();
- // This ensures that waits on global memory aren't introduced on
- // chips that don't have the BackOffBarrier feature enabled in LLVM.
- bool requiresInlineAsm = chipset < kGfx90a;
+ bool hasSplitBarriers = target.has(llvm::AMDGPU::FEAT_GFX12_INSTS);
+ // Inline assembly is only needed for cases where the hardware doesn't use
+ // split barriers and doesn't have FeatureBackOffBarrier (this is mainly
+ // early gfx9). In that case, we use inline assembly to bypass the
+ // conservative insertion of global memory waits at barriers, since we care
+ // more about performance than having debug watches work correctly.
+ bool requiresInlineAsm =
+ !hasSplitBarriers && !target.has(llvm::AMDGPU::FEAT_BACK_OFF_BARRIER);
Attribute mmra =
rewriter.getAttr<LLVM::MMRATagAttr>("amdgpu-synchronize-as", "local");
@@ -669,7 +624,7 @@ struct LDSBarrierOpLowering : public ConvertOpToLLVMPattern<LDSBarrierOp> {
/*convergent=*/false,
/*asm_dialect=*/asmDialectAttr,
/*operand_attrs=*/ArrayAttr());
- } else if (chipset.majorVersion < 12) {
+ } else if (!hasSplitBarriers) {
ROCDL::SBarrierOp::create(rewriter, loc);
} else {
ROCDL::BarrierSignalOp::create(rewriter, loc, -1);
@@ -685,10 +640,11 @@ struct LDSBarrierOpLowering : public ConvertOpToLLVMPattern<LDSBarrierOp> {
};
struct SchedBarrierOpLowering : public ConvertOpToLLVMPattern<SchedBarrierOp> {
- SchedBarrierOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<SchedBarrierOp>(converter), chipset(chipset) {}
+ SchedBarrierOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<SchedBarrierOp>(converter), target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(SchedBarrierOp op, SchedBarrierOp::Adaptor adaptor,
@@ -904,31 +860,34 @@ static void wmmaPushOutputOperand(ConversionPatternRewriter &rewriter,
}
/// Return true if `type` is the E5M2 variant of an 8-bit float that is
-/// supported by the `_bf8` instructions on the given `chipset`.
-static bool typeIsExpectedBf8ForChipset(Chipset chipset, Type type) {
- return (chipset == kGfx942 && isa<Float8E5M2FNUZType>(type)) ||
- (hasOcpFp8(chipset) && isa<Float8E5M2Type>(type));
+/// supported by the `_bf8` instructions on `target`.
+static bool typeIsExpectedBf8ForTarget(const ROCDL::TargetInfo &target,
+ Type type) {
+ return (target.hasFnuzFp8() && isa<Float8E5M2FNUZType>(type)) ||
+ (target.hasOcpFp8() && isa<Float8E5M2Type>(type));
}
/// Return true if `type` is the E4M3FN variant of an 8-bit float that is
-/// supported by the `_fp8` instructions on the given `chipset`.
-static bool typeIsExpectedFp8ForChipset(Chipset chipset, Type type) {
- return (chipset == kGfx942 && isa<Float8E4M3FNUZType>(type)) ||
- (hasOcpFp8(chipset) && isa<Float8E4M3FNType>(type));
+/// supported by the `_fp8` instructions on `target`.
+static bool typeIsExpectedFp8ForTarget(const ROCDL::TargetInfo &target,
+ Type type) {
+ return (target.hasFnuzFp8() && isa<Float8E4M3FNUZType>(type)) ||
+ (target.hasOcpFp8() && isa<Float8E4M3FNType>(type));
}
/// Return the `rocdl` intrinsic corresponding to a MFMA operation `mfma`
/// if one exists. This includes checking to ensure the intrinsic is supported
/// on the architecture you are compiling for.
-static std::optional<StringRef> mfmaOpToIntrinsic(MFMAOp mfma,
- Chipset chipset) {
+static std::optional<StringRef>
+mfmaOpToIntrinsic(MFMAOp mfma, const ROCDL::TargetInfo &target) {
uint32_t m = mfma.getM(), n = mfma.getN(), k = mfma.getK(),
b = mfma.getBlocks();
Type sourceElem = getElementTypeOrSelf(mfma.getSourceA().getType());
Type destElem = getElementTypeOrSelf(mfma.getDestC().getType());
if (sourceElem.isF32() && destElem.isF32()) {
- if (mfma.getReducePrecision() && chipset >= kGfx942) {
+ if (mfma.getReducePrecision() &&
+ target.has(llvm::AMDGPU::FEAT_XF32_INSTS)) {
if (m == 32 && n == 32 && k == 4 && b == 1)
return ROCDL::mfma_f32_32x32x4_xf32::getOperationName();
if (m == 16 && n == 16 && k == 8 && b == 1)
@@ -947,7 +906,7 @@ static std::optional<StringRef> mfmaOpToIntrinsic(MFMAOp mfma,
}
if (sourceElem.isF16() && destElem.isF32()) {
- if (chipset >= kGfx950) {
+ if (target.has(llvm::AMDGPU::FEAT_GFX950_INSTS)) {
if (m == 32 && n == 32 && k == 16 && b == 1)
return ROCDL::mfma_f32_32x32x16_f16::getOperationName();
if (m == 16 && n == 16 && k == 32 && b == 1)
@@ -966,13 +925,13 @@ static std::optional<StringRef> mfmaOpToIntrinsic(MFMAOp mfma,
}
if (sourceElem.isBF16() && destElem.isF32()) {
- if (chipset >= kGfx950) {
+ if (target.has(llvm::AMDGPU::FEAT_GFX950_INSTS)) {
if (m == 32 && n == 32 && k == 16 && b == 1)
return ROCDL::mfma_f32_32x32x16_bf16::getOperationName();
if (m == 16 && n == 16 && k == 32 && b == 1)
return ROCDL::mfma_f32_16x16x32_bf16::getOperationName();
}
- if (chipset >= kGfx90a) {
+ if (target.has(llvm::AMDGPU::FEAT_GFX90A_INSTS)) {
if (m == 32 && n == 32 && k == 4 && b == 2)
return ROCDL::mfma_f32_32x32x4bf16_1k::getOperationName();
if (m == 16 && n == 16 && k == 4 && b == 4)
@@ -997,7 +956,7 @@ static std::optional<StringRef> mfmaOpToIntrinsic(MFMAOp mfma,
}
if (sourceElem.isInteger(8) && destElem.isInteger(32)) {
- if (chipset >= kGfx950) {
+ if (target.has(llvm::AMDGPU::FEAT_GFX950_INSTS)) {
if (m == 32 && n == 32 && k == 32 && b == 1)
return ROCDL::mfma_i32_32x32x32_i8::getOperationName();
if (m == 16 && n == 16 && k == 64 && b == 1)
@@ -1013,51 +972,54 @@ static std::optional<StringRef> mfmaOpToIntrinsic(MFMAOp mfma,
return ROCDL::mfma_i32_32x32x8i8::getOperationName();
if (m == 16 && n == 16 && k == 16 && b == 1)
return ROCDL::mfma_i32_16x16x16i8::getOperationName();
- if (m == 32 && n == 32 && k == 16 && b == 1 && chipset >= kGfx942)
+ if (m == 32 && n == 32 && k == 16 && b == 1 &&
+ target.has(llvm::AMDGPU::FEAT_GFX940_INSTS))
return ROCDL::mfma_i32_32x32x16_i8::getOperationName();
- if (m == 16 && n == 16 && k == 32 && b == 1 && chipset >= kGfx942)
+ if (m == 16 && n == 16 && k == 32 && b == 1 &&
+ target.has(llvm::AMDGPU::FEAT_GFX940_INSTS))
return ROCDL::mfma_i32_16x16x32_i8::getOperationName();
}
- if (sourceElem.isF64() && destElem.isF64() && chipset >= kGfx90a) {
+ if (sourceElem.isF64() && destElem.isF64() &&
+ target.has(llvm::AMDGPU::FEAT_GFX90A_INSTS)) {
if (m == 16 && n == 16 && k == 4 && b == 1)
return ROCDL::mfma_f64_16x16x4f64::getOperationName();
if (m == 4 && n == 4 && k == 4 && b == 4)
return ROCDL::mfma_f64_4x4x4f64::getOperationName();
}
- if (destElem.isF32() && typeIsExpectedBf8ForChipset(chipset, sourceElem)) {
+ if (destElem.isF32() && typeIsExpectedBf8ForTarget(target, sourceElem)) {
// Known to be correct because there are no scalar f8 instructions and
// because a length mismatch will have been caught by the verifier.
Type sourceBElem =
cast<VectorType>(mfma.getSourceB().getType()).getElementType();
if (m == 16 && n == 16 && k == 32 && b == 1) {
- if (typeIsExpectedBf8ForChipset(chipset, sourceBElem))
+ if (typeIsExpectedBf8ForTarget(target, sourceBElem))
return ROCDL::mfma_f32_16x16x32_bf8_bf8::getOperationName();
- if (typeIsExpectedFp8ForChipset(chipset, sourceBElem))
+ if (typeIsExpectedFp8ForTarget(target, sourceBElem))
return ROCDL::mfma_f32_16x16x32_bf8_fp8::getOperationName();
}
if (m == 32 && n == 32 && k == 16 && b == 1) {
- if (typeIsExpectedBf8ForChipset(chipset, sourceBElem))
+ if (typeIsExpectedBf8ForTarget(target, sourceBElem))
return ROCDL::mfma_f32_32x32x16_bf8_bf8::getOperationName();
- if (typeIsExpectedFp8ForChipset(chipset, sourceBElem))
+ if (typeIsExpectedFp8ForTarget(target, sourceBElem))
return ROCDL::mfma_f32_32x32x16_bf8_fp8::getOperationName();
}
}
- if (destElem.isF32() && typeIsExpectedFp8ForChipset(chipset, sourceElem)) {
+ if (destElem.isF32() && typeIsExpectedFp8ForTarget(target, sourceElem)) {
Type sourceBElem =
cast<VectorType>(mfma.getSourceB().getType()).getElementType();
if (m == 16 && n == 16 && k == 32 && b == 1) {
- if (typeIsExpectedBf8ForChipset(chipset, sourceBElem))
+ if (typeIsExpectedBf8ForTarget(target, sourceBElem))
return ROCDL::mfma_f32_16x16x32_fp8_bf8::getOperationName();
- if (typeIsExpectedFp8ForChipset(chipset, sourceBElem))
+ if (typeIsExpectedFp8ForTarget(target, sourceBElem))
return ROCDL::mfma_f32_16x16x32_fp8_fp8::getOperationName();
}
if (m == 32 && n == 32 && k == 16 && b == 1) {
- if (typeIsExpectedBf8ForChipset(chipset, sourceBElem))
+ if (typeIsExpectedBf8ForTarget(target, sourceBElem))
return ROCDL::mfma_f32_32x32x16_fp8_bf8::getOperationName();
- if (typeIsExpectedFp8ForChipset(chipset, sourceBElem))
+ if (typeIsExpectedFp8ForTarget(target, sourceBElem))
return ROCDL::mfma_f32_32x32x16_fp8_fp8::getOperationName();
}
}
@@ -1089,12 +1051,13 @@ using ScaledMFMAIntrinsic =
static std::optional<ScaledMFMAIntrinsic>
mfmaOpToScaledIntrinsic(Type aType, Type bType, Type destType, uint32_t m,
- uint32_t n, uint32_t k, uint32_t b, Chipset chipset) {
+ uint32_t n, uint32_t k, uint32_t b,
+ const ROCDL::TargetInfo &target) {
aType = getElementTypeOrSelf(aType);
bType = getElementTypeOrSelf(bType);
destType = getElementTypeOrSelf(destType);
- if (chipset < kGfx950)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX950_INSTS))
return std::nullopt;
if (!isa<Float32Type>(destType))
return std::nullopt;
@@ -1118,19 +1081,19 @@ mfmaOpToScaledIntrinsic(Type aType, Type bType, Type destType, uint32_t m,
}
static std::optional<ScaledMFMAIntrinsic>
-mfmaOpToScaledIntrinsic(MFMAOp mfma, Chipset chipset) {
+mfmaOpToScaledIntrinsic(MFMAOp mfma, const ROCDL::TargetInfo &target) {
return mfmaOpToScaledIntrinsic(
mfma.getSourceA().getType(), mfma.getSourceB().getType(),
mfma.getDestC().getType(), mfma.getM(), mfma.getN(), mfma.getK(),
- mfma.getBlocks(), chipset);
+ mfma.getBlocks(), target);
}
static std::optional<ScaledMFMAIntrinsic>
-mfmaOpToScaledIntrinsic(ScaledMFMAOp smfma, Chipset chipset) {
+mfmaOpToScaledIntrinsic(ScaledMFMAOp smfma, const ROCDL::TargetInfo &target) {
return mfmaOpToScaledIntrinsic(smfma.getSourceA().getType(),
smfma.getSourceB().getType(),
smfma.getDestC().getType(), smfma.getM(),
- smfma.getN(), smfma.getK(), 1u, chipset);
+ smfma.getN(), smfma.getK(), 1u, target);
}
/// Returns the `rocdl` intrinsic corresponding to a WMMA operation `wmma`
@@ -1285,11 +1248,11 @@ static std::optional<StringRef> wmmaOpToIntrinsicGfx1250(Type elemSourceType,
/// Returns the `rocdl` intrinsic corresponding to a SparseMFMA (smfmac)
/// operation if one exists. This includes checking to ensure the intrinsic is
/// supported on the architecture you are compiling for.
-static std::optional<StringRef> smfmacOpToIntrinsic(SparseMFMAOp op,
- Chipset chipset) {
- bool isGfx950 = chipset >= kGfx950;
- auto isFp8 = [&](Type t) { return typeIsExpectedFp8ForChipset(chipset, t); };
- auto isBf8 = [&](Type t) { return typeIsExpectedBf8ForChipset(chipset, t); };
+static std::optional<StringRef>
+smfmacOpToIntrinsic(SparseMFMAOp op, const ROCDL::TargetInfo &target) {
+ bool isGfx950 = target.has(llvm::AMDGPU::FEAT_GFX950_INSTS);
+ auto isFp8 = [&](Type t) { return typeIsExpectedFp8ForTarget(target, t); };
+ auto isBf8 = [&](Type t) { return typeIsExpectedBf8ForTarget(target, t); };
uint32_t m = op.getM(), n = op.getN(), k = op.getK();
Type sourceAElem = getElementTypeOrSelf(op.getSourceA().getType());
@@ -1384,8 +1347,8 @@ static std::optional<StringRef> smfmacOpToIntrinsic(SparseMFMAOp op,
/// Returns the `rocdl` intrinsic corresponding to a WMMA operation `wmma`
/// if one exists. This includes checking to ensure the intrinsic is supported
/// on the architecture you are compiling for.
-static std::optional<StringRef> wmmaOpToIntrinsic(WMMAOp wmma,
- Chipset chipset) {
+static std::optional<StringRef>
+wmmaOpToIntrinsic(WMMAOp wmma, const ROCDL::TargetInfo &target) {
auto sourceVectorType = cast<VectorType>(wmma.getSourceA().getType());
auto sourceBVectorType = cast<VectorType>(wmma.getSourceB().getType());
auto destVectorType = cast<VectorType>(wmma.getDestC().getType());
@@ -1394,8 +1357,9 @@ static std::optional<StringRef> wmmaOpToIntrinsic(WMMAOp wmma,
Type elemDestType = destVectorType.getElementType();
const uint32_t k = wmma.getK();
- const bool isRDNA3 = chipset.majorVersion == 11;
- const bool isRDNA4 = chipset.majorVersion == 12 && chipset.minorVersion == 0;
+ const bool isRDNA3 = target.isGeneration(11);
+ const bool isRDNA4 =
+ target.isGeneration(12) && !target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS);
// Handle RDNA3 and RDNA4.
if (isRDNA3 || isRDNA4)
@@ -1403,7 +1367,7 @@ static std::optional<StringRef> wmmaOpToIntrinsic(WMMAOp wmma,
k, isRDNA3);
// Handle gfx1250.
- if (chipset == kGfx1250)
+ if (target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return wmmaOpToIntrinsicGfx1250(elemSourceType, elemBSourceType,
elemDestType, k);
@@ -1421,7 +1385,7 @@ struct SparseWMMAOpInfo {
};
static std::optional<SparseWMMAOpInfo>
-sparseWMMAOpToIntrinsic(SparseWMMAOp swmmac, Chipset chipset) {
+sparseWMMAOpToIntrinsic(SparseWMMAOp swmmac, const ROCDL::TargetInfo &target) {
Type sourceAElem = getElementTypeOrSelf(swmmac.getSourceA().getType());
Type sourceBElem = getElementTypeOrSelf(swmmac.getSourceB().getType());
Type destElem = getElementTypeOrSelf(swmmac.getDestC().getType());
@@ -1431,7 +1395,8 @@ sparseWMMAOpToIntrinsic(SparseWMMAOp swmmac, Chipset chipset) {
if ((m != 16) || (n != 16))
return std::nullopt;
- const bool isRDNA4 = chipset.majorVersion == 12 && chipset.minorVersion == 0;
+ const bool isRDNA4 =
+ target.isGeneration(12) && !target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS);
if (isRDNA4) {
if (k == 32) {
if (destElem.isF32() && sourceAElem.isF16() && sourceBElem.isF16())
@@ -1489,7 +1454,7 @@ sparseWMMAOpToIntrinsic(SparseWMMAOp swmmac, Chipset chipset) {
}
}
- const bool isGFX1250 = chipset == kGfx1250;
+ const bool isGFX1250 = target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS);
const bool isWavesize64 = swmmac.getWave64();
if (isGFX1250 && !isWavesize64) {
if (k == 64) {
@@ -1567,10 +1532,11 @@ sparseWMMAOpToIntrinsic(SparseWMMAOp swmmac, Chipset chipset) {
namespace {
struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
- MFMAOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<MFMAOp>(converter), chipset(chipset) {}
+ MFMAOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<MFMAOp>(converter), target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(MFMAOp op, MFMAOpAdaptor adaptor,
@@ -1583,18 +1549,18 @@ struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
if (outVecType.getElementType().isBF16())
intrinsicOutType = outVecType.clone(rewriter.getI16Type());
- if (chipset.majorVersion != 9 || chipset < kGfx908)
+ if (!target.has(llvm::AMDGPU::FEAT_MAI_INSTS))
return op->emitOpError("MFMA only supported on gfx908+");
uint32_t getBlgpField = static_cast<uint32_t>(op.getBlgp());
if (op.getNegateA() || op.getNegateB() || op.getNegateC()) {
- if (chipset < kGfx942)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX940_INSTS))
return op.emitOpError("negation unsupported on older than gfx942");
getBlgpField |=
op.getNegateA() | (op.getNegateB() << 1) | (op.getNegateC() << 2);
}
- std::optional<StringRef> maybeIntrinsic = mfmaOpToIntrinsic(op, chipset);
+ std::optional<StringRef> maybeIntrinsic = mfmaOpToIntrinsic(op, target);
std::optional<ScaledMFMAIntrinsic> maybeScaledIntrinsic =
- mfmaOpToScaledIntrinsic(op, chipset);
+ mfmaOpToScaledIntrinsic(op, target);
if (!maybeIntrinsic.has_value() && !maybeScaledIntrinsic.has_value())
return op.emitOpError("no intrinsic matching MFMA size on given chipset");
@@ -1612,7 +1578,7 @@ struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
// Determine if we can use bf16 in the intrinsic. Newer MFMAs in gfx950+
// allows bf16 as the input. For reference check IntrinsicsAMDGPU.td file.
bool allowBf16 = [&]() {
- if (chipset < kGfx950)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX950_INSTS))
return false;
if (isScaled)
return true;
@@ -1660,10 +1626,11 @@ struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
};
struct ScaledMFMAOpLowering : public ConvertOpToLLVMPattern<ScaledMFMAOp> {
- ScaledMFMAOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern(converter), chipset(chipset) {}
+ ScaledMFMAOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern(converter), target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(ScaledMFMAOp op, ScaledMFMAOpAdaptor adaptor,
@@ -1671,10 +1638,10 @@ struct ScaledMFMAOpLowering : public ConvertOpToLLVMPattern<ScaledMFMAOp> {
Location loc = op.getLoc();
Type intrinsicOutType = typeConverter->convertType(op.getDestD().getType());
- if (chipset.majorVersion != 9 || chipset < kGfx950)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX950_INSTS))
return op->emitOpError("scaled MFMA only supported on gfx908+");
std::optional<ScaledMFMAIntrinsic> maybeScaledIntrinsic =
- mfmaOpToScaledIntrinsic(op, chipset);
+ mfmaOpToScaledIntrinsic(op, target);
if (!maybeScaledIntrinsic.has_value())
return op.emitOpError(
"no intrinsic matching scaled MFMA size on given chipset");
@@ -1706,10 +1673,11 @@ struct ScaledMFMAOpLowering : public ConvertOpToLLVMPattern<ScaledMFMAOp> {
};
struct SparseMFMAOpLowering : public ConvertOpToLLVMPattern<SparseMFMAOp> {
- SparseMFMAOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<SparseMFMAOp>(converter), chipset(chipset) {}
+ SparseMFMAOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<SparseMFMAOp>(converter), target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(SparseMFMAOp op, SparseMFMAOpAdaptor adaptor,
@@ -1721,10 +1689,10 @@ struct SparseMFMAOpLowering : public ConvertOpToLLVMPattern<SparseMFMAOp> {
return rewriter.notifyMatchFailure(op, "type conversion failed");
// smfmac is supported on gfx942 and gfx950.
- if (chipset.majorVersion != 9 || chipset < kGfx942)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX940_INSTS))
return op->emitOpError("sparse MFMA (smfmac) only supported on gfx942+");
- std::optional<StringRef> maybeIntrinsic = smfmacOpToIntrinsic(op, chipset);
+ std::optional<StringRef> maybeIntrinsic = smfmacOpToIntrinsic(op, target);
if (!maybeIntrinsic.has_value())
return op.emitOpError(
"no intrinsic matching sparse MFMA on the given chipset");
@@ -1733,7 +1701,8 @@ struct SparseMFMAOpLowering : public ConvertOpToLLVMPattern<SparseMFMAOp> {
ROCDL::smfmac_f32_16x16x32_bf16::getOperationName() ||
*maybeIntrinsic ==
ROCDL::smfmac_f32_32x32x16_bf16::getOperationName());
- bool isGfx950 = (chipset >= kGfx950) && !isGfx942BF16;
+ bool isGfx950 =
+ (target.has(llvm::AMDGPU::FEAT_GFX950_INSTS)) && !isGfx942BF16;
Value a = convertPackedVectorOperand(rewriter, loc, adaptor.getSourceA(),
isGfx950);
@@ -1761,10 +1730,11 @@ struct SparseMFMAOpLowering : public ConvertOpToLLVMPattern<SparseMFMAOp> {
};
struct WMMAOpLowering : public ConvertOpToLLVMPattern<WMMAOp> {
- WMMAOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<WMMAOp>(converter), chipset(chipset) {}
+ WMMAOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<WMMAOp>(converter), target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(WMMAOp op, WMMAOpAdaptor adaptor,
@@ -1775,10 +1745,10 @@ struct WMMAOpLowering : public ConvertOpToLLVMPattern<WMMAOp> {
if (!outType)
return rewriter.notifyMatchFailure(op, "type conversion failed");
- if (chipset.majorVersion != 11 && chipset.majorVersion != 12)
+ if (!target.isGeneration(11) && !target.isGeneration(12))
return op->emitOpError("WMMA only supported on gfx11 and gfx12");
- bool isGFX1250 = chipset >= kGfx1250;
+ bool isGFX1250 = target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS);
// The WMMA operations represent vectors of bf16s as vectors of i16s
// (except on gfx1250), so we need to bitcast bfloats to i16 and then
@@ -1806,12 +1776,13 @@ struct WMMAOpLowering : public ConvertOpToLLVMPattern<WMMAOp> {
destC = LLVM::BitcastOp::create(
rewriter, loc, destCType.clone(rewriter.getI16Type()), destC);
- std::optional<StringRef> maybeIntrinsic = wmmaOpToIntrinsic(op, chipset);
+ std::optional<StringRef> maybeIntrinsic = wmmaOpToIntrinsic(op, target);
if (!maybeIntrinsic.has_value())
return op.emitOpError("no intrinsic matching WMMA on the given chipset");
- if (chipset.majorVersion >= 12 && op.getSubwordOffset() != 0)
+ if (target.has(llvm::AMDGPU::FEAT_GFX12_INSTS) &&
+ op.getSubwordOffset() != 0)
return op.emitOpError("subwordOffset not supported on gfx12+");
SmallVector<Value, 4> operands;
@@ -1850,7 +1821,7 @@ enum class DotFamily {
};
static std::optional<std::pair<StringRef, DotFamily>>
-dotOpToIntrinsic(DotOp op, Chipset chipset) {
+dotOpToIntrinsic(DotOp op, const ROCDL::TargetInfo &target) {
Type aElem = cast<VectorType>(op.getSourceA().getType()).getElementType();
Type bElem = cast<VectorType>(op.getSourceB().getType()).getElementType();
Type dest = op.getDestC().getType();
@@ -1859,18 +1830,18 @@ dotOpToIntrinsic(DotOp op, Chipset chipset) {
// f16 x f16 -> f32 / f16.
if (aElem.isF16() && bElem.isF16()) {
- if (dest.isF32() && hasDot10Insts(chipset))
+ if (dest.isF32() && target.has(llvm::AMDGPU::FEAT_DOT10_INSTS))
return {{ROCDL::fdot2::getOperationName(), DotFamily::Clamp}};
- if (dest.isF16() && hasDot9Insts(chipset))
+ if (dest.isF16() && target.has(llvm::AMDGPU::FEAT_DOT9_INSTS))
return {{ROCDL::fdot2_f16_f16::getOperationName(), DotFamily::NoClamp}};
return std::nullopt;
}
// bf16 x bf16 -> f32 / bf16.
if (aElem.isBF16() && bElem.isBF16()) {
- if (dest.isF32() && hasDot12Insts(chipset))
+ if (dest.isF32() && target.has(llvm::AMDGPU::FEAT_DOT12_INSTS))
return {{ROCDL::fdot2_f32_bf16::getOperationName(), DotFamily::Clamp}};
- if (dest.isBF16() && hasDot9Insts(chipset))
+ if (dest.isBF16() && target.has(llvm::AMDGPU::FEAT_DOT9_INSTS))
return {{ROCDL::fdot2_bf16_bf16::getOperationName(), DotFamily::NoClamp}};
return std::nullopt;
}
@@ -1882,7 +1853,7 @@ dotOpToIntrinsic(DotOp op, Chipset chipset) {
unsigned elemWidth = aElem.getIntOrFloatBitWidth();
if (mixedSign) {
- if (!hasDot8Insts(chipset))
+ if (!target.has(llvm::AMDGPU::FEAT_DOT8_INSTS))
return std::nullopt;
StringRef name;
switch (elemWidth) {
@@ -1902,19 +1873,21 @@ dotOpToIntrinsic(DotOp op, Chipset chipset) {
bool supported = false;
switch (elemWidth) {
case 16:
- supported = hasDot2Insts(chipset);
+ supported = target.has(llvm::AMDGPU::FEAT_DOT2_INSTS);
name = uA ? ROCDL::udot2::getOperationName()
: ROCDL::sdot2::getOperationName();
break;
case 8:
- supported = uA ? hasDot7Insts(chipset)
- : hasDot1Insts(chipset) || hasDot8Insts(chipset);
+ supported = uA ? target.has(llvm::AMDGPU::FEAT_DOT7_INSTS)
+ : target.has(llvm::AMDGPU::FEAT_DOT1_INSTS) ||
+ target.has(llvm::AMDGPU::FEAT_DOT8_INSTS);
name = uA ? ROCDL::udot4::getOperationName()
: ROCDL::sdot4::getOperationName();
break;
case 4:
- supported = uA ? hasDot7Insts(chipset)
- : hasDot1Insts(chipset) || hasDot8Insts(chipset);
+ supported = uA ? target.has(llvm::AMDGPU::FEAT_DOT7_INSTS)
+ : target.has(llvm::AMDGPU::FEAT_DOT1_INSTS) ||
+ target.has(llvm::AMDGPU::FEAT_DOT8_INSTS);
name = uA ? ROCDL::udot8::getOperationName()
: ROCDL::sdot8::getOperationName();
break;
@@ -1932,7 +1905,7 @@ dotOpToIntrinsic(DotOp op, Chipset chipset) {
bool bIsFp8 = isa<Float8E4M3FNType>(bElem);
bool bIsBf8 = isa<Float8E5M2Type>(bElem);
if ((aIsFp8 || aIsBf8) && (bIsFp8 || bIsBf8) && dest.isF32()) {
- if (!hasDot11Insts(chipset))
+ if (!target.has(llvm::AMDGPU::FEAT_DOT11_INSTS))
return std::nullopt;
StringRef name;
if (aIsFp8 && bIsFp8)
@@ -1950,10 +1923,11 @@ dotOpToIntrinsic(DotOp op, Chipset chipset) {
}
struct DotOpLowering : public ConvertOpToLLVMPattern<DotOp> {
- DotOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<DotOp>(converter), chipset(chipset) {}
+ DotOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<DotOp>(converter), target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(DotOp op, DotOpAdaptor adaptor,
@@ -1961,7 +1935,7 @@ struct DotOpLowering : public ConvertOpToLLVMPattern<DotOp> {
Location loc = op.getLoc();
std::optional<std::pair<StringRef, DotFamily>> maybeIntrinsic =
- dotOpToIntrinsic(op, chipset);
+ dotOpToIntrinsic(op, target);
if (!maybeIntrinsic)
return op.emitOpError("no intrinsic matching dot on the given chipset: ")
<< op.getSourceA().getType() << " * " << op.getSourceB().getType()
@@ -1998,10 +1972,11 @@ struct DotOpLowering : public ConvertOpToLLVMPattern<DotOp> {
};
struct SparseWMMAOpLowering : public ConvertOpToLLVMPattern<SparseWMMAOp> {
- SparseWMMAOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<SparseWMMAOp>(converter), chipset(chipset) {}
+ SparseWMMAOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<SparseWMMAOp>(converter), target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(SparseWMMAOp op, SparseWMMAOpAdaptor adaptor,
@@ -2013,7 +1988,7 @@ struct SparseWMMAOpLowering : public ConvertOpToLLVMPattern<SparseWMMAOp> {
return rewriter.notifyMatchFailure(op, "type conversion failed");
std::optional<SparseWMMAOpInfo> maybeIntrinsic =
- sparseWMMAOpToIntrinsic(op, chipset);
+ sparseWMMAOpToIntrinsic(op, target);
if (!maybeIntrinsic.has_value())
return op.emitOpError(
@@ -2045,8 +2020,7 @@ struct SparseWMMAOpLowering : public ConvertOpToLLVMPattern<SparseWMMAOp> {
if (intrinsic.useClamp && op.getClampAttr())
attrs.push_back({"clamp", op.getClampAttr()});
- const bool isGFX1250orHigher =
- chipset.majorVersion == 12 && chipset.minorVersion >= 5;
+ const bool isGFX1250orHigher = target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS);
Value a = convertPackedVectorOperand(rewriter, loc, adaptor.getSourceA(),
isGFX1250orHigher);
Value b = convertPackedVectorOperand(rewriter, loc, adaptor.getSourceB(),
@@ -2079,10 +2053,11 @@ struct SparseWMMAOpLowering : public ConvertOpToLLVMPattern<SparseWMMAOp> {
};
struct ScaledWMMAOpLowering : public ConvertOpToLLVMPattern<ScaledWMMAOp> {
- ScaledWMMAOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<ScaledWMMAOp>(converter), chipset(chipset) {}
+ ScaledWMMAOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<ScaledWMMAOp>(converter), target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(ScaledWMMAOp op, ScaledWMMAOpAdaptor adaptor,
@@ -2093,7 +2068,7 @@ struct ScaledWMMAOpLowering : public ConvertOpToLLVMPattern<ScaledWMMAOp> {
if (!outType)
return rewriter.notifyMatchFailure(op, "type conversion failed");
- if (chipset < kGfx1250)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op->emitOpError("WMMA scale only supported on gfx1250+");
int64_t m = op.getM();
@@ -2199,15 +2174,17 @@ struct ScaledWMMAOpLowering : public ConvertOpToLLVMPattern<ScaledWMMAOp> {
struct TransposeLoadOpLowering
: public ConvertOpToLLVMPattern<TransposeLoadOp> {
- TransposeLoadOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<TransposeLoadOp>(converter), chipset(chipset) {}
+ TransposeLoadOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<TransposeLoadOp>(converter), target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(TransposeLoadOp op, TransposeLoadOpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset != kGfx950 && chipset < kGfx1250)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX950_INSTS) &&
+ !target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op.emitOpError(
"transpose_load is only supported on gfx950 and gfx1250+");
@@ -2248,7 +2225,7 @@ struct TransposeLoadOpLowering
};
Value intrinsic;
- if (chipset >= kGfx1250) {
+ if (target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS)) {
switch (elementTypeSize) {
case 4: {
if (numElements != 16)
@@ -2348,17 +2325,17 @@ struct TransposeLoadOpLowering
struct GlobalTransposeLoadOpLowering
: public ConvertOpToLLVMPattern<GlobalTransposeLoadOp> {
GlobalTransposeLoadOpLowering(const LLVMTypeConverter &converter,
- Chipset chipset)
+ const ROCDL::TargetInfo &target)
: ConvertOpToLLVMPattern<GlobalTransposeLoadOp>(converter),
- chipset(chipset) {}
+ target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(GlobalTransposeLoadOp op,
GlobalTransposeLoadOpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset < kGfx1200)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX12_INSTS))
return op.emitOpError(
"global_transpose_load is only supported on gfx1200+");
@@ -2386,7 +2363,7 @@ struct GlobalTransposeLoadOpLowering
switch (elementTypeSize) {
case 4: {
assert(numElements == 16);
- if (chipset < kGfx1250)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op.emitOpError("4-bit global_transpose_load requires gfx1250+");
auto rocdlOp = ROCDL::GlobalLoadTr4_B64::create(
rewriter, loc, rocdlResultType, srcPtr, ArrayAttr{}, ArrayAttr{},
@@ -2396,7 +2373,7 @@ struct GlobalTransposeLoadOpLowering
}
case 6: {
assert(numElements == 16);
- if (chipset < kGfx1250)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op.emitOpError("6-bit global_transpose_load requires gfx1250+");
auto rocdlOp = ROCDL::GlobalLoadTr6_B96::create(
rewriter, loc, rocdlResultType, srcPtr, ArrayAttr{}, ArrayAttr{},
@@ -2427,15 +2404,16 @@ struct GlobalTransposeLoadOpLowering
};
struct GatherToLDSOpLowering : public ConvertOpToLLVMPattern<GatherToLDSOp> {
- GatherToLDSOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<GatherToLDSOp>(converter), chipset(chipset) {}
+ GatherToLDSOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<GatherToLDSOp>(converter), target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(GatherToLDSOp op, GatherToLDSOpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset.majorVersion < 9 || chipset.majorVersion > 10)
+ if (!target.has(llvm::AMDGPU::FEAT_VMEM_TO_LDS_LOAD_INSTS))
return op.emitOpError("pre-gfx9 and post-gfx10 not supported");
Location loc = op.getLoc();
@@ -2460,7 +2438,8 @@ struct GatherToLDSOpLowering : public ConvertOpToLLVMPattern<GatherToLDSOp> {
if (!llvm::is_contained({1, 2, 4, 12, 16}, loadWidth))
return op.emitOpError("chipset unsupported element size");
- if (chipset != kGfx950 && llvm::is_contained({12, 16}, loadWidth))
+ if (!target.has(llvm::AMDGPU::FEAT_GFX950_INSTS) &&
+ llvm::is_contained({12, 16}, loadWidth))
return op.emitOpError("Gather to LDS instructions with 12-byte and "
"16-byte load widths are only supported on gfx950");
@@ -2492,17 +2471,17 @@ struct GatherToLDSOpLowering : public ConvertOpToLLVMPattern<GatherToLDSOp> {
struct GlobalLoadAsyncToLDSOpLowering
: public ConvertOpToLLVMPattern<GlobalLoadAsyncToLDSOp> {
GlobalLoadAsyncToLDSOpLowering(const LLVMTypeConverter &converter,
- Chipset chipset)
+ const ROCDL::TargetInfo &target)
: ConvertOpToLLVMPattern<GlobalLoadAsyncToLDSOp>(converter),
- chipset(chipset) {}
+ target(target) {}
- Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(GlobalLoadAsyncToLDSOp op,
GlobalLoadAsyncToLDSOpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset < kGfx1250)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op.emitOpError(
"global_load_async_to_lds is only supported on gfx1250+");
@@ -2569,10 +2548,11 @@ struct GlobalLoadAsyncToLDSOpLowering
namespace {
struct ExtPackedFp8OpLowering final
: public ConvertOpToLLVMPattern<ExtPackedFp8Op> {
- ExtPackedFp8OpLowering(const LLVMTypeConverter &converter, Chipset chipset)
+ ExtPackedFp8OpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
: ConvertOpToLLVMPattern<amdgpu::ExtPackedFp8Op>(converter),
- chipset(chipset) {}
- Chipset chipset;
+ target(target) {}
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(ExtPackedFp8Op op, ExtPackedFp8OpAdaptor adaptor,
@@ -2582,10 +2562,10 @@ struct ExtPackedFp8OpLowering final
struct ScaledExtPackedMatrixOpLowering final
: public ConvertOpToLLVMPattern<ScaledExtPackedMatrixOp> {
ScaledExtPackedMatrixOpLowering(const LLVMTypeConverter &converter,
- Chipset chipset)
+ const ROCDL::TargetInfo &target)
: ConvertOpToLLVMPattern<amdgpu::ScaledExtPackedMatrixOp>(converter),
- chipset(chipset) {}
- Chipset chipset;
+ target(target) {}
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(ScaledExtPackedMatrixOp op,
@@ -2596,10 +2576,10 @@ struct ScaledExtPackedMatrixOpLowering final
struct PackedTrunc2xFp8OpLowering final
: public ConvertOpToLLVMPattern<PackedTrunc2xFp8Op> {
PackedTrunc2xFp8OpLowering(const LLVMTypeConverter &converter,
- Chipset chipset)
+ const ROCDL::TargetInfo &target)
: ConvertOpToLLVMPattern<amdgpu::PackedTrunc2xFp8Op>(converter),
- chipset(chipset) {}
- Chipset chipset;
+ target(target) {}
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(PackedTrunc2xFp8Op op, PackedTrunc2xFp8OpAdaptor adaptor,
@@ -2609,10 +2589,10 @@ struct PackedTrunc2xFp8OpLowering final
struct PackedStochRoundFp8OpLowering final
: public ConvertOpToLLVMPattern<PackedStochRoundFp8Op> {
PackedStochRoundFp8OpLowering(const LLVMTypeConverter &converter,
- Chipset chipset)
+ const ROCDL::TargetInfo &target)
: ConvertOpToLLVMPattern<amdgpu::PackedStochRoundFp8Op>(converter),
- chipset(chipset) {}
- Chipset chipset;
+ target(target) {}
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(PackedStochRoundFp8Op op,
@@ -2622,10 +2602,11 @@ struct PackedStochRoundFp8OpLowering final
struct ScaledExtPackedOpLowering final
: public ConvertOpToLLVMPattern<ScaledExtPackedOp> {
- ScaledExtPackedOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
+ ScaledExtPackedOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
: ConvertOpToLLVMPattern<amdgpu::ScaledExtPackedOp>(converter),
- chipset(chipset) {}
- Chipset chipset;
+ target(target) {}
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(ScaledExtPackedOp op, ScaledExtPackedOpAdaptor adaptor,
@@ -2635,10 +2616,10 @@ struct ScaledExtPackedOpLowering final
struct PackedScaledTruncOpLowering final
: public ConvertOpToLLVMPattern<PackedScaledTruncOp> {
PackedScaledTruncOpLowering(const LLVMTypeConverter &converter,
- Chipset chipset)
+ const ROCDL::TargetInfo &target)
: ConvertOpToLLVMPattern<amdgpu::PackedScaledTruncOp>(converter),
- chipset(chipset) {}
- Chipset chipset;
+ target(target) {}
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(PackedScaledTruncOp op, PackedScaledTruncOpAdaptor adaptor,
@@ -2651,7 +2632,7 @@ LogicalResult ExtPackedFp8OpLowering::matchAndRewrite(
ExtPackedFp8Op op, ExtPackedFp8OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const {
Location loc = op.getLoc();
- if (!(chipset == kGfx942 || hasOcpFp8(chipset)))
+ if (!target.has(llvm::AMDGPU::FEAT_FP8_CONVERSION_INSTS))
return rewriter.notifyMatchFailure(
loc, "Fp8 conversion instructions are not available on target "
"architecture and their emulation is not implemented");
@@ -2682,18 +2663,18 @@ LogicalResult ExtPackedFp8OpLowering::matchAndRewrite(
}
Value i32Source = LLVM::BitcastOp::create(rewriter, loc, i32, source);
if (resultVecType) {
- if (typeIsExpectedBf8ForChipset(chipset, sourceElemType)) {
+ if (typeIsExpectedBf8ForTarget(target, sourceElemType)) {
rewriter.replaceOpWithNewOp<ROCDL::CvtPkF32Bf8Op>(op, f32, i32Source,
op.getIndex());
- } else if (typeIsExpectedFp8ForChipset(chipset, sourceElemType)) {
+ } else if (typeIsExpectedFp8ForTarget(target, sourceElemType)) {
rewriter.replaceOpWithNewOp<ROCDL::CvtPkF32Fp8Op>(op, f32, i32Source,
op.getIndex());
}
} else {
- if (typeIsExpectedBf8ForChipset(chipset, sourceElemType)) {
+ if (typeIsExpectedBf8ForTarget(target, sourceElemType)) {
rewriter.replaceOpWithNewOp<ROCDL::CvtF32Bf8Op>(op, f32, i32Source,
op.getIndex());
- } else if (typeIsExpectedFp8ForChipset(chipset, sourceElemType)) {
+ } else if (typeIsExpectedFp8ForTarget(target, sourceElemType)) {
rewriter.replaceOpWithNewOp<ROCDL::CvtF32Fp8Op>(op, f32, i32Source,
op.getIndex());
}
@@ -2801,7 +2782,7 @@ LogicalResult ScaledExtPackedMatrixOpLowering::matchAndRewrite(
using fp6 = Float6E2M3FNType;
using bf6 = Float6E3M2FNType;
Location loc = op.getLoc();
- if (chipset != kGfx1250) {
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS)) {
return rewriter.notifyMatchFailure(
loc,
"Scaled fp packed conversion instructions are not available on target "
@@ -2872,7 +2853,7 @@ LogicalResult ScaledExtPackedOpLowering::matchAndRewrite(
ScaledExtPackedOp op, ScaledExtPackedOpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const {
Location loc = op.getLoc();
- if (chipset != kGfx950)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX950_INSTS))
return rewriter.notifyMatchFailure(
loc, "Scaled fp conversion instructions are not available on target "
"architecture and their emulation is not implemented");
@@ -2952,7 +2933,7 @@ LogicalResult PackedScaledTruncOpLowering::matchAndRewrite(
PackedScaledTruncOp op, PackedScaledTruncOpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const {
Location loc = op.getLoc();
- if (chipset != kGfx950)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX950_INSTS))
return rewriter.notifyMatchFailure(
loc, "Scaled fp conversion instructions are not available on target "
"architecture and their emulation is not implemented");
@@ -3034,7 +3015,7 @@ LogicalResult PackedTrunc2xFp8OpLowering::matchAndRewrite(
PackedTrunc2xFp8Op op, PackedTrunc2xFp8OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const {
Location loc = op.getLoc();
- if (!(chipset == kGfx942 || hasOcpFp8(chipset)))
+ if (!target.has(llvm::AMDGPU::FEAT_FP8_CONVERSION_INSTS))
return rewriter.notifyMatchFailure(
loc, "Fp8 conversion instructions are not available on target "
"architecture and their emulation is not implemented");
@@ -3054,10 +3035,10 @@ LogicalResult PackedTrunc2xFp8OpLowering::matchAndRewrite(
existing = LLVM::UndefOp::create(rewriter, loc, i32);
Value result;
- if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
+ if (typeIsExpectedBf8ForTarget(target, resultElemType))
result = ROCDL::CvtPkBf8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
existing, op.getWordIndex());
- else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
+ else if (typeIsExpectedFp8ForTarget(target, resultElemType))
result = ROCDL::CvtPkFp8F32Op::create(rewriter, loc, i32, sourceA, sourceB,
existing, op.getWordIndex());
else
@@ -3073,7 +3054,7 @@ LogicalResult PackedStochRoundFp8OpLowering::matchAndRewrite(
PackedStochRoundFp8Op op, PackedStochRoundFp8OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const {
Location loc = op.getLoc();
- if (!(chipset == kGfx942 || hasOcpFp8(chipset)))
+ if (!target.has(llvm::AMDGPU::FEAT_FP8_CONVERSION_INSTS))
return rewriter.notifyMatchFailure(
loc, "Fp8 conversion instructions are not available on target "
"architecture and their emulation is not implemented");
@@ -3091,10 +3072,10 @@ LogicalResult PackedStochRoundFp8OpLowering::matchAndRewrite(
existing = LLVM::UndefOp::create(rewriter, loc, i32);
Value result;
- if (typeIsExpectedBf8ForChipset(chipset, resultElemType))
+ if (typeIsExpectedBf8ForTarget(target, resultElemType))
result = ROCDL::CvtSrBf8F32Op::create(rewriter, loc, i32, source, stoch,
existing, op.getStoreIndex());
- else if (typeIsExpectedFp8ForChipset(chipset, resultElemType))
+ else if (typeIsExpectedFp8ForTarget(target, resultElemType))
result = ROCDL::CvtSrFp8F32Op::create(rewriter, loc, i32, source, stoch,
existing, op.getStoreIndex());
else
@@ -3109,9 +3090,10 @@ LogicalResult PackedStochRoundFp8OpLowering::matchAndRewrite(
// Implement the AMDGPU_DPPLowering class that will convert the amdgpu.dpp
// operation into the corresponding ROCDL instructions.
struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
- AMDGPUDPPLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<DPPOp>(converter), chipset(chipset) {}
- Chipset chipset;
+ AMDGPUDPPLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<DPPOp>(converter), target(target) {}
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(DPPOp DppOp, DPPOp::Adaptor adaptor,
@@ -3295,20 +3277,25 @@ struct AMDGPUSwizzleBitModeLowering
struct AMDGPUPermlaneLowering : public ConvertOpToLLVMPattern<PermlaneSwapOp> {
using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
- AMDGPUPermlaneLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<PermlaneSwapOp>(converter), chipset(chipset) {}
- Chipset chipset;
+ AMDGPUPermlaneLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<PermlaneSwapOp>(converter), target(target) {}
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(PermlaneSwapOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset < kGfx950)
- return op->emitOpError("permlane_swap is only supported on gfx950+");
+ unsigned rowLength = op.getRowLength();
+ bool supported = rowLength == 16
+ ? target.has(llvm::AMDGPU::FEAT_PERMLANE16_SWAP)
+ : target.has(llvm::AMDGPU::FEAT_PERMLANE32_SWAP);
+ if (!supported)
+ return op->emitOpError("permlane_swap of row length ")
+ << rowLength << " is not supported on " << target.getArchName();
Location loc = op.getLoc();
Type i32 = rewriter.getI32Type();
Value src = adaptor.getSrc();
- unsigned rowLength = op.getRowLength();
bool fi = op.getFetchInactive();
bool boundctrl = op.getBoundCtrl();
@@ -3355,14 +3342,15 @@ struct AMDGPUPermlaneVarLowering
: public ConvertOpToLLVMPattern<PermlaneVarOp> {
using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
- AMDGPUPermlaneVarLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<PermlaneVarOp>(converter), chipset(chipset) {}
- Chipset chipset;
+ AMDGPUPermlaneVarLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<PermlaneVarOp>(converter), target(target) {}
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(PermlaneVarOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset < kGfx1200)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX12_INSTS))
return op->emitOpError("permlane_var is only supported on GFX12+");
Location loc = op.getLoc();
@@ -3412,15 +3400,16 @@ constexpr int32_t kDsBarrierPendingCountMask =
struct DsBarrierInitOpLowering
: public ConvertOpToLLVMPattern<DsBarrierInitOp> {
- Chipset chipset;
+ ROCDL::TargetInfo target;
- DsBarrierInitOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<DsBarrierInitOp>(converter), chipset(chipset) {}
+ DsBarrierInitOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<DsBarrierInitOp>(converter), target(target) {}
LogicalResult
matchAndRewrite(DsBarrierInitOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset < kGfx1250)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op->emitOpError("only supported on gfx1250+");
Location loc = op.getLoc();
@@ -3465,17 +3454,17 @@ struct DsBarrierInitOpLowering
struct DsBarrierPollStateOpLowering
: public ConvertOpToLLVMPattern<DsBarrierPollStateOp> {
- Chipset chipset;
+ ROCDL::TargetInfo target;
DsBarrierPollStateOpLowering(const LLVMTypeConverter &converter,
- Chipset chipset)
+ const ROCDL::TargetInfo &target)
: ConvertOpToLLVMPattern<DsBarrierPollStateOp>(converter),
- chipset(chipset) {}
+ target(target) {}
LogicalResult
matchAndRewrite(DsBarrierPollStateOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset < kGfx1250)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op->emitOpError("only supported on gfx1250+");
Location loc = op.getLoc();
@@ -3498,17 +3487,17 @@ struct DsBarrierPollStateOpLowering
struct DsAsyncBarrierArriveOpLowering
: public ConvertOpToLLVMPattern<DsAsyncBarrierArriveOp> {
- Chipset chipset;
+ ROCDL::TargetInfo target;
DsAsyncBarrierArriveOpLowering(const LLVMTypeConverter &converter,
- Chipset chipset)
+ const ROCDL::TargetInfo &target)
: ConvertOpToLLVMPattern<DsAsyncBarrierArriveOp>(converter),
- chipset(chipset) {}
+ target(target) {}
LogicalResult
matchAndRewrite(DsAsyncBarrierArriveOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset < kGfx1250)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op->emitOpError("only supported on gfx1250+");
Location loc = op.getLoc();
@@ -3526,16 +3515,16 @@ struct DsAsyncBarrierArriveOpLowering
struct DsBarrierArriveOpLowering
: public ConvertOpToLLVMPattern<DsBarrierArriveOp> {
- Chipset chipset;
+ ROCDL::TargetInfo target;
- DsBarrierArriveOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<DsBarrierArriveOp>(converter), chipset(chipset) {
- }
+ DsBarrierArriveOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<DsBarrierArriveOp>(converter), target(target) {}
LogicalResult
matchAndRewrite(DsBarrierArriveOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset < kGfx1250)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op->emitOpError("only supported on gfx1250+");
Location loc = op.getLoc();
@@ -3668,14 +3657,15 @@ struct AMDGPUMakeDmaBaseLowering : public ConvertOpToLLVMPattern<BaseOp> {
using ConvertOpToLLVMPattern<BaseOp>::ConvertOpToLLVMPattern;
using Adaptor = typename ConvertOpToLLVMPattern<BaseOp>::OpAdaptor;
- AMDGPUMakeDmaBaseLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<BaseOp>(converter), chipset(chipset) {}
- Chipset chipset;
+ AMDGPUMakeDmaBaseLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<BaseOp>(converter), target(target) {}
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(BaseOp op, Adaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset < kGfx1250)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op->emitOpError("make_dma_base is only supported on gfx1250");
Location loc = op.getLoc();
@@ -3758,9 +3748,10 @@ struct AMDGPULowerDescriptor : public ConvertOpToLLVMPattern<DescriptorOp> {
using ConvertOpToLLVMPattern<DescriptorOp>::ConvertOpToLLVMPattern;
using OpAdaptor = typename ConvertOpToLLVMPattern<DescriptorOp>::OpAdaptor;
- AMDGPULowerDescriptor(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<DescriptorOp>(converter), chipset(chipset) {}
- Chipset chipset;
+ AMDGPULowerDescriptor(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<DescriptorOp>(converter), target(target) {}
+ ROCDL::TargetInfo target;
Value getDGroup0(OpAdaptor &adaptor) const { return adaptor.getBase(); }
@@ -4443,7 +4434,7 @@ struct AMDGPULowerDescriptor : public ConvertOpToLLVMPattern<DescriptorOp> {
LogicalResult
matchAndRewrite(DescriptorOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset < kGfx1250)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op->emitOpError(
"make_dma_descriptor is only supported on gfx1250");
@@ -4469,14 +4460,14 @@ struct AMDGPUTensorLoadStoreOpLowering
using ConvertOpToLLVMPattern<SourceOp>::ConvertOpToLLVMPattern;
using Adaptor = typename ConvertOpToLLVMPattern<SourceOp>::OneToNOpAdaptor;
AMDGPUTensorLoadStoreOpLowering(const LLVMTypeConverter &converter,
- Chipset chipset)
- : ConvertOpToLLVMPattern<SourceOp>(converter), chipset(chipset) {}
- Chipset chipset;
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<SourceOp>(converter), target(target) {}
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(SourceOp op, Adaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset < kGfx1250)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op->emitOpError("is only supported on gfx1250");
ValueRange desc = adaptor.getDesc();
@@ -4496,13 +4487,14 @@ struct AMDGPUTensorLoadStoreOpLowering
struct GlobalPrefetchOpLowering
: public ConvertOpToLLVMPattern<GlobalPrefetchOp> {
- GlobalPrefetchOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<GlobalPrefetchOp>(converter), chipset(chipset) {}
+ GlobalPrefetchOpLowering(const LLVMTypeConverter &converter,
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<GlobalPrefetchOp>(converter), target(target) {}
LogicalResult
matchAndRewrite(GlobalPrefetchOp op, GlobalPrefetchOpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset < kGfx1250)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op->emitOpError("is only supported on gfx1250+");
const bool isSpeculative = op.getSpeculative();
@@ -4532,7 +4524,7 @@ struct GlobalPrefetchOpLowering
}
private:
- Chipset chipset;
+ ROCDL::TargetInfo target;
};
struct ConvertAMDGPUToROCDLPass
@@ -4541,16 +4533,16 @@ struct ConvertAMDGPUToROCDLPass
void runOnOperation() override {
MLIRContext *ctx = &getContext();
- FailureOr<Chipset> maybeChipset = Chipset::parse(chipset);
- if (failed(maybeChipset)) {
- emitError(UnknownLoc::get(ctx), "Invalid chipset name: " + chipset);
+ FailureOr<ROCDL::TargetInfo> targetInfo = ROCDL::TargetInfo::get(
+ ROCDL::resolveArchOption(arch, chipset),
+ /*waveSize=*/0, [&] { return emitError(UnknownLoc::get(ctx)); });
+ if (failed(targetInfo))
return signalPassFailure();
- }
RewritePatternSet patterns(ctx);
LLVMTypeConverter converter(ctx);
- populateAMDGPUToROCDLConversionPatterns(converter, patterns, *maybeChipset);
+ populateAMDGPUToROCDLConversionPatterns(converter, patterns, *targetInfo);
amdgpu::populateCommonGPUTypeAndAttributeConversions(converter);
LLVMConversionTarget target(getContext());
target.addIllegalDialect<::mlir::amdgpu::AMDGPUDialect>();
@@ -4642,9 +4634,9 @@ void mlir::populateAMDGPUTypeAndAttributeConversions(
typeConverter.addTargetMaterialization(addUnrealizedCast);
}
-void mlir::populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter,
- RewritePatternSet &patterns,
- Chipset chipset) {
+void mlir::populateAMDGPUToROCDLConversionPatterns(
+ LLVMTypeConverter &converter, RewritePatternSet &patterns,
+ const ROCDL::TargetInfo &target) {
populateAMDGPUTypeAndAttributeConversions(converter);
patterns
.add<FatRawBufferCastLowering,
@@ -4679,7 +4671,7 @@ void mlir::populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter,
ROCDL::TensorStoreFromLDSOp>,
DsBarrierInitOpLowering, DsBarrierPollStateOpLowering,
DsAsyncBarrierArriveOpLowering, DsBarrierArriveOpLowering,
- GlobalPrefetchOpLowering>(converter, chipset);
+ GlobalPrefetchOpLowering>(converter, target);
patterns.add<AMDGPUSwizzleBitModeLowering, DsBarrierStatePhaseOpLowering,
DsBarrierStatePendingCountOpLowering,
DsBarrierStateInitCountOpLowering,
diff --git a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
index d0714739589f5c..1bb1cf60f107b7 100644
--- a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
+++ b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
@@ -9,11 +9,11 @@
#include "mlir/Conversion/ArithToAMDGPU/ArithToAMDGPU.h"
#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
-#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Arith/Utils/Utils.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
+#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
#include "mlir/Dialect/Utils/IndexingUtils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
@@ -32,10 +32,6 @@ using namespace mlir;
using namespace mlir::amdgpu;
namespace {
-// Define commonly used chipsets versions for convenience.
-constexpr Chipset kGfx942 = Chipset(9, 4, 2);
-constexpr Chipset kGfx950 = Chipset(9, 5, 0);
-
struct ArithToAMDGPUConversionPass final
: impl::ArithToAMDGPUConversionPassBase<ArithToAMDGPUConversionPass> {
using impl::ArithToAMDGPUConversionPassBase<
@@ -47,10 +43,10 @@ struct ArithToAMDGPUConversionPass final
struct ExtFOnFloat8RewritePattern final : OpRewritePattern<arith::ExtFOp> {
using Base::Base;
- Chipset chipset;
- ExtFOnFloat8RewritePattern(MLIRContext *ctx, Chipset chipset,
+ ROCDL::TargetInfo target;
+ ExtFOnFloat8RewritePattern(MLIRContext *ctx, const ROCDL::TargetInfo &target,
PatternBenefit benefit)
- : OpRewritePattern::OpRewritePattern(ctx, benefit), chipset(chipset) {}
+ : OpRewritePattern::OpRewritePattern(ctx, benefit), target(target) {}
LogicalResult matchAndRewrite(arith::ExtFOp op,
PatternRewriter &rewriter) const override;
@@ -59,10 +55,11 @@ struct ExtFOnFloat8RewritePattern final : OpRewritePattern<arith::ExtFOp> {
struct TruncFToFloat8RewritePattern final : OpRewritePattern<arith::TruncFOp> {
bool saturateFP8 = false;
TruncFToFloat8RewritePattern(MLIRContext *ctx, bool saturateFP8,
- Chipset chipset, PatternBenefit benefit)
+ const ROCDL::TargetInfo &target,
+ PatternBenefit benefit)
: OpRewritePattern::OpRewritePattern(ctx, benefit),
- saturateFP8(saturateFP8), chipset(chipset) {}
- Chipset chipset;
+ saturateFP8(saturateFP8), target(target) {}
+ ROCDL::TargetInfo target;
LogicalResult matchAndRewrite(arith::TruncFOp op,
PatternRewriter &rewriter) const override;
@@ -92,13 +89,12 @@ struct ScalingTruncFRewritePattern final
LogicalResult matchAndRewrite(arith::ScalingTruncFOp op,
PatternRewriter &rewriter) const override;
};
-
} // end namespace
-static bool isSupportedF8(Type elementType, Chipset chipset) {
- if (chipset == kGfx942)
+static bool isSupportedF8(Type elementType, const ROCDL::TargetInfo &target) {
+ if (target.hasFnuzFp8())
return isa<Float8E4M3FNUZType, Float8E5M2FNUZType>(elementType);
- if (hasOcpFp8(chipset))
+ if (target.hasOcpFp8())
return isa<Float8E4M3FNType, Float8E5M2Type>(elementType);
return false;
}
@@ -126,7 +122,7 @@ ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
return failure();
inType = inVecType.getElementType();
}
- if (!isSupportedF8(inType, chipset))
+ if (!isSupportedF8(inType, target))
return failure();
Location loc = op.getLoc();
@@ -278,7 +274,7 @@ TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
// Conversion between 8-bit floats is not supported with truncation enabled.
return failure();
- if (!isSupportedF8(outType, chipset))
+ if (!isSupportedF8(outType, target))
return failure();
Location loc = op.getLoc();
@@ -704,13 +700,13 @@ ScalingTruncFRewritePattern::matchAndRewrite(arith::ScalingTruncFOp op,
void mlir::arith::populateArithToAMDGPUConversionPatterns(
RewritePatternSet &patterns, bool convertFP8Arithmetic,
bool saturateFP8Truncf, bool allowPackedF16Rtz, bool supportsScaledExtTrunc,
- Chipset chipset, PatternBenefit benefit) {
+ const ROCDL::TargetInfo &target, PatternBenefit benefit) {
if (convertFP8Arithmetic) {
- patterns.add<ExtFOnFloat8RewritePattern>(patterns.getContext(), chipset,
+ patterns.add<ExtFOnFloat8RewritePattern>(patterns.getContext(), target,
benefit);
patterns.add<TruncFToFloat8RewritePattern>(
- patterns.getContext(), saturateFP8Truncf, chipset, benefit);
+ patterns.getContext(), saturateFP8Truncf, target, benefit);
}
if (allowPackedF16Rtz)
patterns.add<TruncfToFloat16RewritePattern>(patterns.getContext(), benefit);
@@ -725,18 +721,19 @@ void ArithToAMDGPUConversionPass::runOnOperation() {
Operation *op = getOperation();
MLIRContext *ctx = &getContext();
RewritePatternSet patterns(op->getContext());
- FailureOr<amdgpu::Chipset> maybeChipset = amdgpu::Chipset::parse(chipset);
- if (failed(maybeChipset)) {
- emitError(UnknownLoc::get(ctx), "Invalid chipset name: " + chipset);
+ FailureOr<ROCDL::TargetInfo> targetInfo = ROCDL::TargetInfo::get(
+ ROCDL::resolveArchOption(arch, chipset), /*waveSize=*/0,
+ [&] { return emitError(UnknownLoc::get(ctx)); });
+ if (failed(targetInfo))
return signalPassFailure();
- }
bool convertFP8Arithmetic =
- *maybeChipset == kGfx942 || hasOcpFp8(*maybeChipset);
- bool supportsScaledExtTrunc = *maybeChipset == kGfx950;
+ targetInfo->has(llvm::AMDGPU::FEAT_FP8_CONVERSION_INSTS);
+ bool supportsScaledExtTrunc =
+ targetInfo->has(llvm::AMDGPU::FEAT_GFX950_INSTS);
arith::populateArithToAMDGPUConversionPatterns(
patterns, convertFP8Arithmetic, saturateFP8Truncf, allowPackedF16Rtz,
- supportsScaledExtTrunc, *maybeChipset);
+ supportsScaledExtTrunc, *targetInfo);
if (failed(applyPatternsGreedily(op, std::move(patterns))))
return signalPassFailure();
}
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index c6f81fd533dffd..4c98fe247bd123 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -241,18 +241,18 @@ struct GPUSubgroupSizeOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp> {
using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
GPUSubgroupSizeOpToROCDL(const LLVMTypeConverter &converter,
- amdgpu::Chipset chipset)
- : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp>(converter),
- chipset(chipset) {}
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp>(converter), target(target) {
+ }
LogicalResult
matchAndRewrite(gpu::SubgroupSizeOp op, gpu::SubgroupSizeOp::Adaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
LLVM::ConstantRangeAttr bounds = nullptr;
- bool isBeforeGfx10 = chipset.majorVersion < 10;
+ bool isWave64 = target.getWavefrontSize() == 64;
if (auto upperBoundAttr = op.getUpperBoundAttr()) {
bounds = rewriter.getAttr<LLVM::ConstantRangeAttr>(
- /*bitWidth=*/32, /*lower=*/isBeforeGfx10 ? 64 : 32,
+ /*bitWidth=*/32, /*lower=*/isWave64 ? 64 : 32,
/*upper=*/op.getUpperBoundAttr().getInt() + 1);
}
Value wavefrontOp = ROCDL::WavefrontSizeOp::create(
@@ -263,16 +263,15 @@ struct GPUSubgroupSizeOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp> {
return success();
}
- const amdgpu::Chipset chipset;
+ const ROCDL::TargetInfo target;
};
struct GPUSubgroupIdOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupIdOp> {
using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
GPUSubgroupIdOpToROCDL(const LLVMTypeConverter &converter,
- amdgpu::Chipset chipset)
- : ConvertOpToLLVMPattern<gpu::SubgroupIdOp>(converter), chipset(chipset) {
- }
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<gpu::SubgroupIdOp>(converter), target(target) {}
LogicalResult
matchAndRewrite(gpu::SubgroupIdOp op, gpu::SubgroupIdOp::Adaptor adaptor,
@@ -281,7 +280,7 @@ struct GPUSubgroupIdOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupIdOp> {
auto int32Type = rewriter.getI32Type();
Value subgroupId;
- if (chipset.majorVersion >= 12) {
+ if (target.has(llvm::AMDGPU::FEAT_GFX12_INSTS)) {
// For gfx12+, use the hardware wave.id register directly.
LLVM::ConstantRangeAttr bounds;
if (auto upperBoundAttr = op.getUpperBoundAttr())
@@ -345,7 +344,7 @@ struct GPUSubgroupIdOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupIdOp> {
return success();
}
- const amdgpu::Chipset chipset;
+ const ROCDL::TargetInfo target;
};
static bool isSupportedReadLaneType(Type type) {
@@ -567,10 +566,10 @@ static constexpr int32_t kWholeClusterBarrierId = -3;
static constexpr int32_t kWholeWorkgroupBarrierId = -1;
struct GPUBarrierOpLowering final : ConvertOpToLLVMPattern<gpu::BarrierOp> {
GPUBarrierOpLowering(const LLVMTypeConverter &converter,
- amdgpu::Chipset chipset)
- : ConvertOpToLLVMPattern<gpu::BarrierOp>(converter), chipset(chipset) {}
+ const ROCDL::TargetInfo &target)
+ : ConvertOpToLLVMPattern<gpu::BarrierOp>(converter), target(target) {}
- amdgpu::Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(gpu::BarrierOp op, gpu::BarrierOp::Adaptor adaptor,
@@ -591,7 +590,7 @@ struct GPUBarrierOpLowering final : ConvertOpToLLVMPattern<gpu::BarrierOp> {
// Cluster scope: gfx1250+ only, signal/wait with constant -3.
if (scope == gpu::BarrierScope::Cluster) {
- if (chipset < amdgpu::Chipset(12, 5, 0))
+ if (!target.has(llvm::AMDGPU::FEAT_GFX1250_INSTS))
return op.emitOpError("cluster scope barriers require gfx1250+");
emitFences(op.getAddressSpaces(), rewriter, loc, "cluster",
/*before=*/true);
@@ -609,7 +608,7 @@ struct GPUBarrierOpLowering final : ConvertOpToLLVMPattern<gpu::BarrierOp> {
// Named barrier path.
if (Value namedBarrier = adaptor.getNamedBarrier()) {
- if (chipset.majorVersion < 12)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX12_INSTS))
return op.emitOpError("named barriers require gfx12+");
emitFences(op.getAddressSpaces(), rewriter, loc, "workgroup",
@@ -631,7 +630,7 @@ struct GPUBarrierOpLowering final : ConvertOpToLLVMPattern<gpu::BarrierOp> {
// Regular workgroup barrier.
emitFences(op.getAddressSpaces(), rewriter, loc, "workgroup",
/*before=*/true);
- if (chipset.majorVersion < 12) {
+ if (!target.has(llvm::AMDGPU::FEAT_GFX12_INSTS)) {
ROCDL::SBarrierOp::create(rewriter, loc);
} else {
ROCDL::BarrierSignalOp::create(rewriter, loc, kWholeWorkgroupBarrierId);
@@ -648,17 +647,17 @@ struct GPUBarrierOpLowering final : ConvertOpToLLVMPattern<gpu::BarrierOp> {
struct GPUInitializeNamedBarrierOpLowering final
: ConvertOpToLLVMPattern<gpu::InitializeNamedBarrierOp> {
GPUInitializeNamedBarrierOpLowering(const LLVMTypeConverter &converter,
- amdgpu::Chipset chipset)
+ const ROCDL::TargetInfo &target)
: ConvertOpToLLVMPattern<gpu::InitializeNamedBarrierOp>(converter),
- chipset(chipset) {}
+ target(target) {}
- amdgpu::Chipset chipset;
+ ROCDL::TargetInfo target;
LogicalResult
matchAndRewrite(gpu::InitializeNamedBarrierOp op,
gpu::InitializeNamedBarrierOp::Adaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (chipset.majorVersion < 12)
+ if (!target.has(llvm::AMDGPU::FEAT_GFX12_INSTS))
return op.emitOpError("named barriers require gfx12+");
Location loc = op.getLoc();
@@ -757,11 +756,11 @@ struct LowerGpuOpsToROCDLOpsPass final
UnitAttr::get(ctx));
}
- FailureOr<amdgpu::Chipset> maybeChipset = amdgpu::Chipset::parse(chipset);
- if (failed(maybeChipset)) {
- emitError(UnknownLoc::get(ctx), "Invalid chipset name: " + chipset);
+ FailureOr<ROCDL::TargetInfo> targetInfo = ROCDL::TargetInfo::get(
+ ROCDL::resolveArchOption(arch, chipset), waveSize,
+ [&] { return emitError(UnknownLoc::get(ctx)); });
+ if (failed(targetInfo))
return signalPassFailure();
- }
/// Customize the bitwidth used for the device side index computations.
LowerToLLVMOptions options(
@@ -792,7 +791,7 @@ struct LowerGpuOpsToROCDLOpsPass final
{
RewritePatternSet patterns(ctx);
populateGpuRewritePatterns(patterns);
- populateGpuPromoteShuffleToAMDGPUPatterns(patterns, maybeChipset);
+ populateGpuPromoteShuffleToAMDGPUPatterns(patterns, *targetInfo);
(void)applyPatternsGreedily(m, std::move(patterns));
}
@@ -828,9 +827,9 @@ struct LowerGpuOpsToROCDLOpsPass final
}
populateAMDGPUToROCDLConversionPatterns(converter, llvmPatterns,
- *maybeChipset);
+ *targetInfo);
populateGpuToROCDLConversionPatterns(converter, llvmPatterns, runtime,
- *maybeChipset);
+ *targetInfo);
configureGpuToROCDLConversionLegality(target);
if (failed(applyPartialConversion(m, target, std::move(llvmPatterns))))
signalPassFailure();
@@ -878,7 +877,7 @@ void mlir::configureGpuToROCDLConversionLegality(ConversionTarget &target) {
void mlir::populateGpuToROCDLConversionPatterns(
const LLVMTypeConverter &converter, RewritePatternSet &patterns,
- mlir::gpu::amd::Runtime runtime, amdgpu::Chipset chipset) {
+ mlir::gpu::amd::Runtime runtime, const ROCDL::TargetInfo &target) {
using gpu::index_lowering::IndexKind;
using gpu::index_lowering::IntrType;
using mlir::gpu::amd::Runtime;
@@ -917,7 +916,7 @@ void mlir::populateGpuToROCDLConversionPatterns(
GPUSubgroupBroadcastOpToROCDL, GPUBallotOpToROCDL>(converter);
patterns.add<GPUSubgroupIdOpToROCDL, GPUSubgroupSizeOpToROCDL,
GPUBarrierOpLowering, GPUInitializeNamedBarrierOpLowering>(
- converter, chipset);
+ converter, target);
- populateMathToROCDLConversionPatterns(converter, patterns, chipset);
+ populateMathToROCDLConversionPatterns(converter, patterns, target);
}
diff --git a/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp b/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp
index 04c3c9e4136e71..b54de7e87adabd 100644
--- a/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp
+++ b/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp
@@ -11,11 +11,11 @@
#include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
#include "mlir/Conversion/LLVMCommon/VectorPattern.h"
-#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncDialect.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
+#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
#include "mlir/Dialect/Math/IR/Math.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/BuiltinDialect.h"
@@ -85,7 +85,7 @@ struct ClampFOpConversion final
void mlir::populateMathToROCDLConversionPatterns(
const LLVMTypeConverter &converter, RewritePatternSet &patterns,
- std::optional<amdgpu::Chipset> chipset) {
+ std::optional<ROCDL::TargetInfo> target) {
// Handled by mathToLLVM: math::AbsIOp
// Handled by mathToLLVM: math::AbsFOp
// Handled by mathToLLVM: math::CopySignOp
@@ -161,10 +161,10 @@ void mlir::populateMathToROCDLConversionPatterns(
populateOpPatterns<arith::RemFOp>(converter, patterns, "__ocml_fmod_f32",
"__ocml_fmod_f64", "__ocml_fmod_f16");
- if (chipset.has_value() && chipset->majorVersion >= 9) {
+ if (target && target->has(llvm::AMDGPU::FEAT_GFX9_INSTS)) {
patterns.add<ClampFOpConversion>(converter);
} else {
- LDBG() << "Chipset dependent patterns were not added";
+ LDBG() << "Target dependent patterns were not added";
}
}
@@ -184,15 +184,20 @@ void ConvertMathToROCDLPass::runOnOperation() {
LowerToLLVMOptions options(ctx, DataLayout(m));
LLVMTypeConverter converter(ctx, options);
- FailureOr<amdgpu::Chipset> maybeChipset;
- if (!chipset.empty()) {
- maybeChipset = amdgpu::Chipset::parse(chipset);
- if (failed(maybeChipset))
+ // An empty architecture means "no target", in which case the
+ // target-dependent patterns are simply not added.
+ std::optional<ROCDL::TargetInfo> resolved;
+ StringRef resolvedArch = ROCDL::resolveArchOption(arch, chipset);
+ if (!resolvedArch.empty()) {
+ FailureOr<ROCDL::TargetInfo> targetInfo =
+ ROCDL::TargetInfo::get(resolvedArch, /*waveSize=*/0, [&] {
+ return emitError(UnknownLoc::get(&getContext()));
+ });
+ if (failed(targetInfo))
return signalPassFailure();
+ resolved = *targetInfo;
}
- populateMathToROCDLConversionPatterns(
- converter, patterns,
- succeeded(maybeChipset) ? std::optional(*maybeChipset) : std::nullopt);
+ populateMathToROCDLConversionPatterns(converter, patterns, resolved);
ConversionTarget target(getContext());
target
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt b/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
index 29baef635ec802..28fa958181d038 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
@@ -16,6 +16,7 @@ add_mlir_dialect_library(MLIRAMDGPUTransforms
MLIRAffineUtils
MLIRArithDialect
MLIRMemRefDialect
+ MLIRROCDLDialect
MLIRSCFDialect
MLIRVectorDialect
MLIRControlFlowDialect
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
index ce47da8d9ee973..930235995b4e1a 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
@@ -9,9 +9,9 @@
#include "mlir/Dialect/AMDGPU/Transforms/Passes.h"
#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
-#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
+#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/TypeUtilities.h"
@@ -193,43 +193,49 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
return success();
}
+/// Returns whether \p op can be lowered to a native buffer atomic fadd on
+/// \p target.
+static bool isFaddNativelySupported(const ROCDL::TargetInfo &target,
+ RawBufferAtomicFaddOp op) {
+ namespace AMDGPU = ::llvm::AMDGPU;
+
+ // A target with only the no-return form can still perform the atomic, it
+ // just cannot report the old value, so it suffices when the result is dead.
+ bool hasFadd = target.has(AMDGPU::FEAT_ATOMIC_FADD_RTN_INSTS) ||
+ (target.has(AMDGPU::FEAT_ATOMIC_FADD_NO_RTN_INSTS) &&
+ op.getOldValue().use_empty());
+ if (!hasFadd)
+ return false;
+
+ // The packed 16-bit forms are separate instructions with their own features.
+ Type elemType = getElementTypeOrSelf(op.getValue().getType());
+ if (isa<Float16Type>(elemType))
+ return target.has(AMDGPU::FEAT_ATOMIC_BUFFER_GLOBAL_PK_ADD_F16_INSTS);
+ if (isa<BFloat16Type>(elemType))
+ return target.has(AMDGPU::FEAT_ATOMIC_BUFFER_PK_ADD_BF16_INST);
+ return true;
+}
+
void mlir::amdgpu::populateAmdgpuEmulateAtomicsPatterns(
- ConversionTarget &target, RewritePatternSet &patterns, Chipset chipset,
- PatternBenefit benefit) {
- // gfx10 has no atomic adds.
- if (chipset.majorVersion == 10 || chipset < Chipset(9, 0, 8)) {
- target.addIllegalOp<RawBufferAtomicFaddOp>();
- }
- // gfx11 has no fp16 atomics
- if (chipset.majorVersion == 11) {
- target.addDynamicallyLegalOp<RawBufferAtomicFaddOp>(
- [](RawBufferAtomicFaddOp op) -> bool {
- Type elemType = getElementTypeOrSelf(op.getValue().getType());
- return !isa<Float16Type, BFloat16Type>(elemType);
+ ConversionTarget &target, RewritePatternSet &patterns,
+ const ROCDL::TargetInfo &targetInfo, PatternBenefit benefit) {
+ namespace AMDGPU = ::llvm::AMDGPU;
+
+ target.addDynamicallyLegalOp<RawBufferAtomicFaddOp>(
+ [targetInfo](RawBufferAtomicFaddOp op) -> bool {
+ return isFaddNativelySupported(targetInfo, op);
+ });
+
+ // Floating-point min and max are only emulated on gfx9; later generations
+ // have them for every type this op accepts. f64 is the one gfx9 case that
+ // may be native.
+ if (targetInfo.isGeneration(9)) {
+ target.addDynamicallyLegalOp<RawBufferAtomicFmaxOp>(
+ [targetInfo](RawBufferAtomicFmaxOp op) -> bool {
+ return op.getValue().getType().isF64() &&
+ targetInfo.has(AMDGPU::FEAT_ATOMIC_FMIN_FMAX_GLOBAL_F64);
});
}
- // gfx9 has no to a very limited support for floating-point min and max.
- if (chipset.majorVersion == 9) {
- if (chipset >= Chipset(9, 0, 0xa)) {
- // gfx90a supports f64 max (and min, but we don't have a min wrapper right
- // now) but all other types need to be emulated.
- target.addDynamicallyLegalOp<RawBufferAtomicFmaxOp>(
- [](RawBufferAtomicFmaxOp op) -> bool {
- return op.getValue().getType().isF64();
- });
- } else {
- target.addIllegalOp<RawBufferAtomicFmaxOp>();
- }
- // TODO(https://github.com/llvm/llvm-project/issues/129206): Refactor
- // this to avoid hardcoding ISA version: gfx950 has bf16 atomics.
- if (chipset < Chipset(9, 5, 0)) {
- target.addDynamicallyLegalOp<RawBufferAtomicFaddOp>(
- [](RawBufferAtomicFaddOp op) -> bool {
- Type elemType = getElementTypeOrSelf(op.getValue().getType());
- return !isa<BFloat16Type>(elemType);
- });
- }
- }
patterns.add<
RawBufferAtomicByCasPattern<RawBufferAtomicFaddOp, arith::AddFOp>,
RawBufferAtomicByCasPattern<RawBufferAtomicFmaxOp, arith::MaximumFOp>,
@@ -240,11 +246,11 @@ void mlir::amdgpu::populateAmdgpuEmulateAtomicsPatterns(
void AmdgpuEmulateAtomicsPass::runOnOperation() {
Operation *op = getOperation();
- FailureOr<Chipset> maybeChipset = Chipset::parse(chipset);
- if (failed(maybeChipset)) {
- emitError(op->getLoc(), "Invalid chipset name: " + chipset);
+ FailureOr<ROCDL::TargetInfo> targetInfo =
+ ROCDL::TargetInfo::get(ROCDL::resolveArchOption(arch, chipset),
+ /*waveSize=*/0, [&] { return op->emitError(); });
+ if (failed(targetInfo))
return signalPassFailure();
- }
MLIRContext &ctx = getContext();
ConversionTarget target(ctx);
@@ -252,7 +258,7 @@ void AmdgpuEmulateAtomicsPass::runOnOperation() {
target.markUnknownOpDynamicallyLegal(
[](Operation *op) -> bool { return true; });
- populateAmdgpuEmulateAtomicsPatterns(target, patterns, *maybeChipset);
+ populateAmdgpuEmulateAtomicsPatterns(target, patterns, *targetInfo);
if (failed(applyPartialConversion(op, target, std::move(patterns))))
return signalPassFailure();
}
diff --git a/mlir/lib/Dialect/GPU/Pipelines/GPUToROCDLPipeline.cpp b/mlir/lib/Dialect/GPU/Pipelines/GPUToROCDLPipeline.cpp
index 1e5fd09a00a758..edbcaa1f8ca1c0 100644
--- a/mlir/lib/Dialect/GPU/Pipelines/GPUToROCDLPipeline.cpp
+++ b/mlir/lib/Dialect/GPU/Pipelines/GPUToROCDLPipeline.cpp
@@ -28,6 +28,7 @@
#include "mlir/Dialect/GPU/IR/GPUDialect.h"
#include "mlir/Dialect/GPU/Pipelines/Passes.h"
#include "mlir/Dialect/GPU/Transforms/Passes.h"
+#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
#include "mlir/Dialect/MemRef/Transforms/Passes.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Pass/PassOptions.h"
@@ -42,12 +43,13 @@ namespace {
//===----------------------------------------------------------------------===//
void buildCommonPassPipeline(
OpPassManager &pm, const mlir::gpu::GPUToROCDLPipelineOptions &options) {
+ std::string arch = ROCDL::resolveArchOption(options.arch, options.chip).str();
// Lower AMDGPU dialect ops (e.g. amdgpu.lds_barrier, amdgpu.dpp,
// amdgpu.mfma, amdgpu.dot, ...) to ROCDL intrinsics first, while they may
// still live in unout-lined `gpu.launch` bodies. Mirrors the way NVVM's
// pipeline runs `convert-nvgpu-to-nvvm` before kernel outlining.
ConvertAMDGPUToROCDLPassOptions amdgpuToROCDLOpt;
- amdgpuToROCDLOpt.chipset = options.chip;
+ amdgpuToROCDLOpt.arch = arch;
pm.addPass(createConvertAMDGPUToROCDLPass(amdgpuToROCDLOpt));
pm.addPass(createGpuKernelOutliningPass());
@@ -57,12 +59,10 @@ void buildCommonPassPipeline(
pm.addPass(memref::createExpandStridedMetadataPass());
GpuROCDLAttachTargetOptions rocdlTargetOptions;
- rocdlTargetOptions.triple = options.triple;
- rocdlTargetOptions.chip = options.chip;
- rocdlTargetOptions.features = options.features;
+ rocdlTargetOptions.arch = arch;
rocdlTargetOptions.abiVersion = options.abiVersion;
rocdlTargetOptions.optLevel = options.optLevel;
- rocdlTargetOptions.wave64Flag = options.wave64;
+ rocdlTargetOptions.waveSize = options.waveSize;
pm.addPass(createGpuROCDLAttachTarget(rocdlTargetOptions));
pm.addPass(createLowerAffinePass());
@@ -79,8 +79,10 @@ void buildCommonPassPipeline(
//===----------------------------------------------------------------------===//
void buildGpuPassPipeline(OpPassManager &pm,
const mlir::gpu::GPUToROCDLPipelineOptions &options) {
+ std::string arch = ROCDL::resolveArchOption(options.arch, options.chip).str();
ConvertGpuOpsToROCDLOpsOptions opt;
- opt.chipset = options.chip;
+ opt.arch = arch;
+ opt.waveSize = options.waveSize;
opt.useBarePtrCallConv = options.kernelUseBarePtrCallConv;
opt.indexBitwidth = options.indexBitWidth;
// Always declare HIP as the runtime so that gpu.printf etc. lower to the
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index 9591b76a5330ce..a7858781a10792 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -14,7 +14,6 @@
#include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h"
#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
-#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/GPU/IR/GPUDialect.h"
#include "mlir/Dialect/GPU/TransformOps/Utils.h"
@@ -108,21 +107,20 @@ void transform::ApplyGPUToROCDLConversionPatternsOp::populatePatterns(
TypeConverter &typeConverter, RewritePatternSet &patterns) {
auto &llvmTypeConverter = static_cast<LLVMTypeConverter &>(typeConverter);
amdgpu::populateCommonGPUTypeAndAttributeConversions(llvmTypeConverter);
- FailureOr<amdgpu::Chipset> maybeChipset =
- amdgpu::Chipset::parse(getChipset());
- assert(llvm::succeeded(maybeChipset) && "expected valid chipset");
+ // The verifier has already rejected anything unparseable.
+ FailureOr<ROCDL::TargetInfo> targetInfo =
+ ROCDL::TargetInfo::get(getArch(), getWavesize().value_or(0));
+ assert(llvm::succeeded(targetInfo) && "verifier accepted this target");
populateGpuToROCDLConversionPatterns(
- llvmTypeConverter, patterns, mlir::gpu::amd::Runtime::HIP, *maybeChipset);
+ llvmTypeConverter, patterns, mlir::gpu::amd::Runtime::HIP, *targetInfo);
}
LogicalResult
transform::ApplyGPUToROCDLConversionPatternsOp::verifyTypeConverter(
transform::TypeConverterBuilderOpInterface builder) {
- FailureOr<amdgpu::Chipset> maybeChipset =
- amdgpu::Chipset::parse(getChipset());
- if (failed(maybeChipset)) {
- return emitOpError("Invalid chipset name: " + getChipset());
- }
+ if (failed(ROCDL::TargetInfo::get(getArch(), getWavesize().value_or(0),
+ [&] { return emitOpError(); })))
+ return failure();
if (builder.getTypeConverterType() != "LLVMTypeConverter")
return emitOpError("expected LLVMTypeConverter");
return success();
@@ -138,16 +136,27 @@ void ApplyGPURewritePatternsOp::populatePatterns(RewritePatternSet &patterns) {
void transform::ApplyGPUPromoteShuffleToAMDGPUPatternsOp::populatePatterns(
RewritePatternSet &patterns) {
- std::optional<StringRef> chipsetName = getChipset();
- std::optional<amdgpu::Chipset> maybeChipset;
- if (chipsetName) {
- FailureOr<amdgpu::Chipset> parsedChipset =
- amdgpu::Chipset::parse(*chipsetName);
- assert(llvm::succeeded(parsedChipset) && "expected valid chipset");
- maybeChipset = parsedChipset;
+ std::optional<StringRef> archName = getArch();
+ std::optional<ROCDL::TargetInfo> targetInfo;
+ if (archName) {
+ // The verifier has already rejected anything unparseable.
+ FailureOr<ROCDL::TargetInfo> parsed = ROCDL::TargetInfo::get(*archName);
+ assert(llvm::succeeded(parsed) && "verifier accepted this target");
+ targetInfo = *parsed;
}
- populateGpuPromoteShuffleToAMDGPUPatterns(patterns, maybeChipset);
+ populateGpuPromoteShuffleToAMDGPUPatterns(patterns, targetInfo);
+}
+
+LogicalResult transform::ApplyGPUPromoteShuffleToAMDGPUPatternsOp::verify() {
+ std::optional<StringRef> archName = getArch();
+ if (!archName)
+ return success();
+
+ if (failed(ROCDL::TargetInfo::get(*archName, /*waveSize=*/0,
+ [&] { return emitOpError(); })))
+ return failure();
+ return success();
}
//===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
index 01da26f88a84d0..26c87c5b5d914a 100644
--- a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
@@ -11,8 +11,8 @@
//
//===----------------------------------------------------------------------===//
-#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
#include "mlir/Dialect/GPU/Transforms/Passes.h"
+#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
@@ -23,9 +23,6 @@
using namespace mlir;
namespace {
-
-constexpr amdgpu::Chipset kGfx950 = amdgpu::Chipset(9, 5, 0);
-
/// Try to promote `gpu.shuffle` to `amdgpu.swizzle_bitmode`, width must be 64
/// and offset must be a constant integer in the range [0, 31].
struct PromoteShuffleToSwizzlePattern
@@ -96,14 +93,13 @@ struct PromoteShuffleToPermlanePattern
return success();
}
};
-
} // namespace
void mlir::populateGpuPromoteShuffleToAMDGPUPatterns(
- RewritePatternSet &patterns, std::optional<amdgpu::Chipset> maybeChipset) {
+ RewritePatternSet &patterns, std::optional<ROCDL::TargetInfo> target) {
patterns.add<PromoteShuffleToSwizzlePattern>(patterns.getContext(),
/*benefit*/ 1);
- if (maybeChipset && *maybeChipset >= kGfx950)
+ if (target && target->has(llvm::AMDGPU::FEAT_PERMLANE32_SWAP))
patterns.add<PromoteShuffleToPermlanePattern>(patterns.getContext(),
/*benefit*/ 2);
}
diff --git a/mlir/lib/Dialect/GPU/Transforms/ROCDLAttachTarget.cpp b/mlir/lib/Dialect/GPU/Transforms/ROCDLAttachTarget.cpp
index 1f44ffa52e068e..cef6deee5c3548 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ROCDLAttachTarget.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ROCDLAttachTarget.cpp
@@ -15,6 +15,7 @@
#include "mlir/Dialect/GPU/IR/GPUDialect.h"
#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
+#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
#include "mlir/IR/Builders.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Target/LLVM/ROCDL/Target.h"
@@ -33,7 +34,7 @@ struct ROCDLAttachTarget
: public impl::GpuROCDLAttachTargetBase<ROCDLAttachTarget> {
using Base::Base;
- DictionaryAttr getFlags(OpBuilder &builder) const;
+ DictionaryAttr getFlags(OpBuilder &builder, bool isWave64) const;
void runOnOperation() override;
@@ -43,13 +44,14 @@ struct ROCDLAttachTarget
};
} // namespace
-DictionaryAttr ROCDLAttachTarget::getFlags(OpBuilder &builder) const {
+DictionaryAttr ROCDLAttachTarget::getFlags(OpBuilder &builder,
+ bool isWave64) const {
UnitAttr unitAttr = builder.getUnitAttr();
SmallVector<NamedAttribute, 6> flags;
auto addFlag = [&](StringRef flag) {
flags.push_back(builder.getNamedAttr(flag, unitAttr));
};
- if (!wave64Flag)
+ if (!isWave64)
addFlag("no_wave64");
if (fastFlag)
addFlag("fast");
@@ -68,10 +70,89 @@ DictionaryAttr ROCDLAttachTarget::getFlags(OpBuilder &builder) const {
void ROCDLAttachTarget::runOnOperation() {
OpBuilder builder(&getContext());
+
+ // #rocdl.target feeds the TargetMachine, whose -mcpu is a bare processor
+ // name. Split an `arch` into the pieces the attribute wants, mirroring
+ // Clang.
+ std::string resolvedTriple = triple;
+ std::string resolvedChip = chip;
+ std::string resolvedFeatures = features;
+ // `wavesize` is the only wavefront-size control. Without an `arch` there is
+ // no target to ask, so leaving it at 0 keeps the Wave64 that `#rocdl.target`
+ // has always assumed.
+ bool resolvedWave64 = waveSize != 32;
+ // Set when `arch` was given and used so its xnack/sramecc modifiers can
+ // become module flags.
+ std::optional<ROCDL::TargetInfo> targetInfo;
+ if (!arch.empty()) {
+ std::optional<llvm::AMDGPU::TargetID> id =
+ ROCDL::TargetInfo::parseTargetID(arch);
+ if (!id) {
+ emitError(UnknownLoc::get(&getContext()))
+ << "'" << arch << "' is not a valid AMDGPU architecture";
+ return signalPassFailure();
+ }
+ // #rocdl.target requires a chip, so a triple that names no GPU (the legacy
+ // subarch-less "amdgcn-amd-amdhsa") cannot be attached.
+ if (id->getGPUKind() == llvm::AMDGPU::GK_NONE) {
+ emitError(UnknownLoc::get(&getContext()))
+ << "'" << arch
+ << "' names no GPU; a chip is required to attach a "
+ "target";
+ return signalPassFailure();
+ }
+
+ llvm::Triple parsed(id->getTargetTripleString());
+
+ StringRef archName = parsed.getArchName();
+ llvm::Triple::SubArchType subArch =
+ llvm::AMDGPU::getSubArch(id->getGPUKind());
+ if (StringRef subArchName = llvm::AMDGPU::getSubArchName(subArch);
+ !subArchName.empty())
+ archName = subArchName;
+
+ // Drop the "unknown" environment part of triples since a lot of the
+ // toolchain expects a 3-component form.
+ resolvedTriple =
+ (parsed.getEnvironment() == llvm::Triple::UnknownEnvironment
+ ? llvm::Triple(archName, parsed.getVendorName(),
+ parsed.getOSName())
+ : llvm::Triple(archName, parsed.getVendorName(),
+ parsed.getOSName(), parsed.getEnvironmentName()))
+ .str();
+ resolvedChip = llvm::AMDGPU::getArchNameAMDGCN(id->getGPUKind()).str();
+
+ // Take the wavefront size from the target, since `wavesize`'s Wave64
+ // fallback is wrong on targets like gfx10.
+ FailureOr<ROCDL::TargetInfo> info =
+ ROCDL::TargetInfo::get(arch, waveSize, [&] {
+ return emitError(UnknownLoc::get(&getContext()));
+ });
+ if (failed(info))
+ return signalPassFailure();
+ targetInfo = *info;
+ resolvedWave64 = info->getWavefrontSize() == 64;
+
+ // Record the wavefrontsize option into the features set so that the device
+ // libraries and codegen agree with each other in cases where both
+ // wavefontsize32 and wavefrontsize64 are permitted options.
+ if (info->supportsBothWavefrontSizes()) {
+ if (!resolvedFeatures.empty())
+ resolvedFeatures += ",";
+ resolvedFeatures +=
+ resolvedWave64 ? "+wavefrontsize64" : "+wavefrontsize32";
+ }
+ } else if (waveSize != 0 && waveSize != 32 && waveSize != 64) {
+ emitError(UnknownLoc::get(&getContext()))
+ << "wavefront size must be 32 or 64, got " << waveSize;
+ return signalPassFailure();
+ }
+
ArrayRef<std::string> libs(linkLibs);
SmallVector<StringRef> filesToLink(libs);
auto target = builder.getAttr<ROCDLTargetAttr>(
- optLevel, triple, chip, features, abiVersion, getFlags(builder),
+ optLevel, resolvedTriple, resolvedChip, resolvedFeatures, abiVersion,
+ getFlags(builder, resolvedWave64),
filesToLink.empty() ? nullptr : builder.getStrArrayAttr(filesToLink));
llvm::Regex matcher(moduleMatcher);
for (Region ®ion : getOperation()->getRegions())
@@ -89,5 +170,7 @@ void ROCDLAttachTarget::runOnOperation() {
targets.erase(llvm::unique(targets), targets.end());
// Update the target attribute array.
module.setTargetsAttr(builder.getArrayAttr(targets));
+ if (targetInfo)
+ targetInfo->migrateArchFeaturesToModuleFlags(module);
}
}
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
index ec1571a56fe4a8..23f1a16ca85aa7 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
@@ -11,12 +11,12 @@
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
-#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/GPU/IR/GPUDialect.h"
#include "mlir/Dialect/GPU/Transforms/Passes.h"
#include "mlir/Dialect/GPU/Utils/GPUUtils.h"
#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
+#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/Location.h"
@@ -370,7 +370,8 @@ struct VectorSubgroupReduceToShuffles final
static FailureOr<Value>
createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
Value input, gpu::AllReduceOperation mode,
- const ClusterInfo &ci, amdgpu::Chipset chipset) {
+ const ClusterInfo &ci,
+ const ROCDL::TargetInfo &target) {
Location loc = op.getLoc();
Value dpp;
Value res = input;
@@ -414,7 +415,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
gpu::convertReductionKind(mode), res, dpp);
}
if (ci.clusterSize >= 32) {
- if (chipset.majorVersion <= 9) {
+ if (!target.has(llvm::AMDGPU::FEAT_GFX10_INSTS)) {
// Broadcast last value from each row to next row.
// Use row mask to avoid polluting row 0 (and row 2 if wave-64).
dpp = amdgpu::DPPOp::create(rewriter, loc, res.getType(), res, res,
@@ -449,7 +450,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
/*or_mask=*/31,
/*xor_mask=*/0);
}
- } else if (chipset.majorVersion <= 12) {
+ } else if (!target.has(llvm::AMDGPU::FEAT_GFX13_INSTS)) {
// Use a permute lane to cross rows (row 1 <-> row 0, row 3 <-> row 2).
Value uint32Max = arith::ConstantOp::create(
rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(-1));
@@ -472,7 +473,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
}
}
if (ci.clusterSize >= 64) {
- if (chipset.majorVersion <= 9) {
+ if (!target.has(llvm::AMDGPU::FEAT_GFX10_INSTS)) {
// Broadcast 31st lane value to rows 2 and 3.
dpp = amdgpu::DPPOp::create(rewriter, loc, res.getType(), res, res,
amdgpu::DPPPerm::row_bcast_31,
@@ -486,7 +487,7 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
res =
ROCDL::ReadlaneOp::create(rewriter, loc, res.getType(), res, lane63);
- } else if (chipset.majorVersion <= 12) {
+ } else if (!target.has(llvm::AMDGPU::FEAT_GFX13_INSTS)) {
// Assume reduction across 32 lanes has been done.
// Perform final reduction manually by summing values in lane 0 and
// lane 32.
@@ -516,10 +517,11 @@ createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
struct ScalarSubgroupReduceToDPP final
: OpRewritePattern<gpu::SubgroupReduceOp> {
ScalarSubgroupReduceToDPP(MLIRContext *ctx, unsigned subgroupSize,
- bool matchClustered, amdgpu::Chipset chipset,
+ bool matchClustered,
+ const ROCDL::TargetInfo &target,
PatternBenefit benefit)
: OpRewritePattern(ctx, benefit), subgroupSize(subgroupSize),
- matchClustered(matchClustered), chipset(chipset) {}
+ matchClustered(matchClustered), target(target) {}
LogicalResult matchAndRewrite(gpu::SubgroupReduceOp op,
PatternRewriter &rewriter) const override {
@@ -545,7 +547,7 @@ struct ScalarSubgroupReduceToDPP final
op, "Value type is not a compatible scalar.");
FailureOr<Value> dpp = createSubgroupDPPReduction(
- rewriter, op, op.getValue(), op.getOp(), *ci, chipset);
+ rewriter, op, op.getValue(), op.getOp(), *ci, target);
if (failed(dpp))
return failure();
@@ -556,7 +558,7 @@ struct ScalarSubgroupReduceToDPP final
private:
unsigned subgroupSize = 0;
bool matchClustered = false;
- amdgpu::Chipset chipset;
+ ROCDL::TargetInfo target;
};
} // namespace
@@ -569,18 +571,18 @@ void mlir::populateGpuBreakDownSubgroupReducePatterns(
}
void mlir::populateGpuLowerSubgroupReduceToDPPPatterns(
- RewritePatternSet &patterns, unsigned subgroupSize, amdgpu::Chipset chipset,
- PatternBenefit benefit) {
+ RewritePatternSet &patterns, unsigned subgroupSize,
+ const ROCDL::TargetInfo &target, PatternBenefit benefit) {
patterns.add<ScalarSubgroupReduceToDPP>(patterns.getContext(), subgroupSize,
- /*matchClustered=*/false, chipset,
+ /*matchClustered=*/false, target,
benefit);
}
void mlir::populateGpuLowerClusteredSubgroupReduceToDPPPatterns(
- RewritePatternSet &patterns, unsigned subgroupSize, amdgpu::Chipset chipset,
- PatternBenefit benefit) {
+ RewritePatternSet &patterns, unsigned subgroupSize,
+ const ROCDL::TargetInfo &target, PatternBenefit benefit) {
patterns.add<ScalarSubgroupReduceToDPP>(patterns.getContext(), subgroupSize,
- /*matchClustered=*/true, chipset,
+ /*matchClustered=*/true, target,
benefit);
}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/8-bit-floats-ocp-gfx1170.mlir b/mlir/test/Conversion/AMDGPUToROCDL/8-bit-floats-ocp-gfx1170.mlir
deleted file mode 100644
index a98d6d1e4c2745..00000000000000
--- a/mlir/test/Conversion/AMDGPUToROCDL/8-bit-floats-ocp-gfx1170.mlir
+++ /dev/null
@@ -1,23 +0,0 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1170 --split-input-file --verify-diagnostics
-
-func.func @ext_packed_fp8(%v: vector<4xf8E4M3FN>) -> f32 {
- // expected-error at below {{failed to legalize operation 'amdgpu.ext_packed_fp8'}}
- %ret = amdgpu.ext_packed_fp8 %v[0] : vector<4xf8E4M3FN> to f32
- func.return %ret : f32
-}
-
-// -----
-
-func.func @ext_packed_bf8(%v: vector<4xf8E5M2>) -> f32 {
- // expected-error at below {{failed to legalize operation 'amdgpu.ext_packed_fp8'}}
- %ret = amdgpu.ext_packed_fp8 %v[0] : vector<4xf8E5M2> to f32
- func.return %ret : f32
-}
-
-// -----
-
-func.func @packed_trunc_2xfp8(%v: f32) -> vector<4xf8E4M3FN> {
- // expected-error at below {{failed to legalize operation 'amdgpu.packed_trunc_2xfp8'}}
- %ret = amdgpu.packed_trunc_2xfp8 %v, undef into undef[word 0] : f32 to vector<4xf8E4M3FN>
- func.return %ret : vector<4xf8E4M3FN>
-}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/8-bit-floats-ocp.mlir b/mlir/test/Conversion/AMDGPUToROCDL/8-bit-floats-ocp.mlir
index 464d47216c81be..39c340553180e2 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/8-bit-floats-ocp.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/8-bit-floats-ocp.mlir
@@ -1,5 +1,6 @@
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx950 | FileCheck %s
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx1200 | FileCheck %s
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.50-amd-amdhsa | FileCheck %s
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu12.00-amd-amdhsa | FileCheck %s
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu11.70-amd-amdhsa | FileCheck %s
// CHECK-LABEL: func @ext_scalar
// CHECK: [[V:%.+]] = builtin.unrealized_conversion_cast %{{.+}} : f8E5M2 to i8
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/8-bit-floats.mlir b/mlir/test/Conversion/AMDGPUToROCDL/8-bit-floats.mlir
index 03fcb266a2e870..c0fc9ed9490f7c 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/8-bit-floats.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/8-bit-floats.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx942 | FileCheck %s
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa | FileCheck %s
// CHECK-LABEL: func @ext_scalar
// CHECK: [[V:%.+]] = builtin.unrealized_conversion_cast %{{.+}} : f8E5M2FNUZ to i8
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
index 8086aa788c8add..ef82534362bc21 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
@@ -1,10 +1,10 @@
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx908 | FileCheck %s --check-prefixes=CHECK,RECORDS32,GFX9,GFX908
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx90a | FileCheck %s --check-prefixes=CHECK,RECORDS32,GFX9,GFX90A
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx942 | FileCheck %s --check-prefixes=CHECK,RECORDS32,GFX9,GFX942
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx1030 | FileCheck %s --check-prefixes=CHECK,RECORDS32,GFX10,RDNA
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx1100 | FileCheck %s --check-prefixes=CHECK,RECORDS32,GFX11,RDNA
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx1201 | FileCheck %s --check-prefixes=CHECK,RECORDS32,GFX12,RDNA
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx1250 | FileCheck %s --check-prefixes=CHECK,RECORDS45,GFX1250
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.08-amd-amdhsa | FileCheck %s --check-prefixes=CHECK,RECORDS32,GFX9,GFX908
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.0a-amd-amdhsa | FileCheck %s --check-prefixes=CHECK,RECORDS32,GFX9,GFX90A
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa | FileCheck %s --check-prefixes=CHECK,RECORDS32,GFX9,GFX942
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu10.30-amd-amdhsa | FileCheck %s --check-prefixes=CHECK,RECORDS32,GFX10,RDNA
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu11.00-amd-amdhsa | FileCheck %s --check-prefixes=CHECK,RECORDS32,GFX11,RDNA
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu12.01-amd-amdhsa | FileCheck %s --check-prefixes=CHECK,RECORDS32,GFX12,RDNA
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu12.50-amd-amdhsa | FileCheck %s --check-prefixes=CHECK,RECORDS45,GFX1250
// CHECK: #[[$MMRA_TAG:.+]] = #llvm.mmra_tag<"amdgpu-synchronize-as":"local">
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/dot-gfx11.mlir b/mlir/test/Conversion/AMDGPUToROCDL/dot-gfx11.mlir
index a87227884dc2a9..972577028c3f4b 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/dot-gfx11.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/dot-gfx11.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1100 | FileCheck %s
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu11.00-amd-amdhsa | FileCheck %s
// CHECK-LABEL: @dot_fdot2_f16_f16
func.func @dot_fdot2_f16_f16(%a: vector<2xf16>, %b: vector<2xf16>, %c: f16) -> f16 {
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/dot-gfx12.mlir b/mlir/test/Conversion/AMDGPUToROCDL/dot-gfx12.mlir
index 3213b5fa8f5c2f..44d36eb225fb3f 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/dot-gfx12.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/dot-gfx12.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1200 | FileCheck %s
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu12.00-amd-amdhsa | FileCheck %s
// CHECK-LABEL: @dot_fp8_fp8
func.func @dot_fp8_fp8(%a: vector<4xf8E4M3FN>, %b: vector<4xf8E4M3FN>, %c: f32) -> f32 {
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/dot-gfx9.mlir b/mlir/test/Conversion/AMDGPUToROCDL/dot-gfx9.mlir
index e13a9976974dc6..022f3bc38ffdd4 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/dot-gfx9.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/dot-gfx9.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx906 | FileCheck %s
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu9.06-amd-amdhsa | FileCheck %s
// CHECK-LABEL: @dot_fdot2
func.func @dot_fdot2(%a: vector<2xf16>, %b: vector<2xf16>, %c: f32) -> f32 {
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/dot-invalid.mlir b/mlir/test/Conversion/AMDGPUToROCDL/dot-invalid.mlir
index dd26ab7040734c..810a2e02f01eda 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/dot-invalid.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/dot-invalid.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx906 --split-input-file -verify-diagnostics
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx942 --split-input-file -verify-diagnostics
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu9.06-amd-amdhsa --split-input-file -verify-diagnostics
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa --split-input-file -verify-diagnostics
// fp8 dot4 is only available on gfx12+.
func.func @dot_fp8_requires_gfx12(%a: vector<4xf8E4M3FN>, %b: vector<4xf8E4M3FN>, %c: f32) -> f32 {
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/dpp.mlir b/mlir/test/Conversion/AMDGPUToROCDL/dpp.mlir
index a4c98111c29561..3c5ccdd02b2298 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/dpp.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/dpp.mlir
@@ -1,6 +1,7 @@
-// RUN: mlir-opt -convert-amdgpu-to-rocdl=chipset=gfx908 %s | FileCheck %s
-// RUN: mlir-opt -convert-amdgpu-to-rocdl=chipset=gfx90a %s | FileCheck %s
-// RUN: mlir-opt -convert-amdgpu-to-rocdl=chipset=gfx942 %s | FileCheck %s
+// RUN: mlir-opt -convert-amdgpu-to-rocdl=arch=amdgpu9.08-amd-amdhsa %s | FileCheck %s
+// RUN: mlir-opt -convert-amdgpu-to-rocdl=arch=amdgpu9.0a-amd-amdhsa %s | FileCheck %s
+// RUN: mlir-opt -convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa %s | FileCheck %s
+// RUN: mlir-opt -convert-amdgpu-to-rocdl=arch=amdgpu9.50-amd-amdhsa %s | FileCheck %s
func.func @test_dpp(%arg0: i32, %arg1: i32) -> i32 {
// CHECK-LABEL: func @test_dpp
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
index a98b742c3ce39a..1439e97d074f8e 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1250 --split-input-file --verify-diagnostics \
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu12.50-amd-amdhsa --split-input-file --verify-diagnostics \
// RUN: | FileCheck %s
// CHECK-LABEL: @scaled_ext_packed_matrix_fp4
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/global-prefetch.mlir b/mlir/test/Conversion/AMDGPUToROCDL/global-prefetch.mlir
index f71de64cd071f2..1e226ef65b3c67 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/global-prefetch.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/global-prefetch.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1250 --split-input-file --verify-diagnostics | FileCheck %s
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu12.50-amd-amdhsa --split-input-file --verify-diagnostics | FileCheck %s
// CHECK-LABEL: @glb_prefetch0
func.func @glb_prefetch0(%src : memref<64x64xf16, #gpu.address_space<global>>, %i : i64, %j : i64) {
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/global_transpose_load.mlir b/mlir/test/Conversion/AMDGPUToROCDL/global_transpose_load.mlir
index f378d7232d7b33..a57f2ccfabd4e4 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/global_transpose_load.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/global_transpose_load.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-opt %s --split-input-file --verify-diagnostics -convert-amdgpu-to-rocdl=chipset=gfx1201 | FileCheck %s
-// RUN: mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=chipset=gfx1250 | FileCheck %s --check-prefixes=CHECK,CHECK-GFX1250
-// RUN: not mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=chipset=gfx942 2>&1 | FileCheck %s --check-prefix=CHECK-OLD
+// RUN: mlir-opt %s --split-input-file --verify-diagnostics -convert-amdgpu-to-rocdl=arch=amdgpu12.01-amd-amdhsa | FileCheck %s
+// RUN: mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=arch=amdgpu12.50-amd-amdhsa | FileCheck %s --check-prefixes=CHECK,CHECK-GFX1250
+// RUN: not mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa 2>&1 | FileCheck %s --check-prefix=CHECK-OLD
// CHECK-LABEL: func @global_transpose_load_8xf16
func.func @global_transpose_load_8xf16(%i : index, %j : index,
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/lds-barrier-gfx90c.mlir b/mlir/test/Conversion/AMDGPUToROCDL/lds-barrier-gfx90c.mlir
index 87b93949be51e0..c0eddea2de97c2 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/lds-barrier-gfx90c.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/lds-barrier-gfx90c.mlir
@@ -1,11 +1,12 @@
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx90c | FileCheck %s
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.0c-amd-amdhsa | FileCheck %s
// gfx90c needs the inline assembly workaround, just like gfx908.
// CHECK-LABEL: func @lds_barrier
func.func @lds_barrier() {
// CHECK: llvm.fence syncscope("workgroup") release
- // CHECK-NEXT: rocdl.s.barrier
+ // CHECK-NEXT: llvm.inline_asm has_side_effects asm_dialect = att
+ // CHECK-SAME: ";;;WARNING: BREAKS DEBUG WATCHES\0As_barrier"
// CHECK-NEXT: llvm.fence syncscope("workgroup") acquire
amdgpu.lds_barrier
func.return
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/load_lds-gfx950.mlir b/mlir/test/Conversion/AMDGPUToROCDL/load_lds-gfx950.mlir
index ff8f19c33a437d..092ba971169f10 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/load_lds-gfx950.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/load_lds-gfx950.mlir
@@ -1,5 +1,5 @@
-// RUN: not mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=chipset=gfx942 2>&1 | FileCheck %s --check-prefix=GFX942
-// RUN: mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=chipset=gfx950 | FileCheck %s --check-prefix=GFX950
+// RUN: not mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa 2>&1 | FileCheck %s --check-prefix=GFX942
+// RUN: mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=arch=amdgpu9.50-amd-amdhsa | FileCheck %s --check-prefix=GFX950
// GFX950-LABEL: func @fat_buffer_load_to_rocdl_f96
// GFX950-SAME: (%[[ARG0:.*]]: memref<128x72xf32, #amdgpu.address_space<fat_raw_buffer>>)
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/load_lds.mlir b/mlir/test/Conversion/AMDGPUToROCDL/load_lds.mlir
index c2783c216d66d9..2d749e37aadba7 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/load_lds.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/load_lds.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx942 | FileCheck %s
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx950 | FileCheck %s
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa | FileCheck %s
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu9.50-amd-amdhsa | FileCheck %s
// CHECK-LABEL: func @global_load_to_rocdl_f32
// CHECK-SAME: (%[[ARG0:.*]]: memref<128x72xf32, #gpu.address_space<global>>)
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/memory_counter_wait.mlir b/mlir/test/Conversion/AMDGPUToROCDL/memory_counter_wait.mlir
index 537ef59b503a6e..0b2e29d678474e 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/memory_counter_wait.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/memory_counter_wait.mlir
@@ -1,7 +1,7 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx942 | FileCheck %s --check-prefixes=CHECK,GFX9
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1030 | FileCheck %s --check-prefixes=CHECK,GFX10
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1100 | FileCheck %s --check-prefixes=CHECK,GFX11
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1201 | FileCheck %s --check-prefixes=CHECK,GFX12
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa | FileCheck %s --check-prefixes=CHECK,GFX9
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu10.30-amd-amdhsa | FileCheck %s --check-prefixes=CHECK,GFX10
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu11.00-amd-amdhsa | FileCheck %s --check-prefixes=CHECK,GFX11
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu12.01-amd-amdhsa | FileCheck %s --check-prefixes=CHECK,GFX12
// CHECK-LABEL: func @memory_counter_wait
func.func @memory_counter_wait() {
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/memory_counter_wait_tensor.mlir b/mlir/test/Conversion/AMDGPUToROCDL/memory_counter_wait_tensor.mlir
index 5b29e01abebdbc..8e250bef47ad0d 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/memory_counter_wait_tensor.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/memory_counter_wait_tensor.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1250 | FileCheck %s
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu12.50-amd-amdhsa | FileCheck %s
// CHECK-LABEL: func @memory_counter_wait_tensor
func.func @memory_counter_wait_tensor() {
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/memory_counter_wait_unsupported.mlir b/mlir/test/Conversion/AMDGPUToROCDL/memory_counter_wait_unsupported.mlir
index 1d2f692bee4882..201fa9caf1e14d 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/memory_counter_wait_unsupported.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/memory_counter_wait_unsupported.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-opt %s --verify-diagnostics --convert-amdgpu-to-rocdl=chipset=gfx942
-// RUN: mlir-opt %s --verify-diagnostics --convert-amdgpu-to-rocdl=chipset=gfx1030
-// RUN: mlir-opt %s --verify-diagnostics --convert-amdgpu-to-rocdl=chipset=gfx1100
+// RUN: mlir-opt %s --verify-diagnostics --convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa
+// RUN: mlir-opt %s --verify-diagnostics --convert-amdgpu-to-rocdl=arch=amdgpu10.30-amd-amdhsa
+// RUN: mlir-opt %s --verify-diagnostics --convert-amdgpu-to-rocdl=arch=amdgpu11.00-amd-amdhsa
func.func @memory_counter_wait_tensor() {
// expected-error @below{{failed to legalize operation 'amdgpu.memory_counter_wait'}}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/mfma-fp8-invalid.mlir b/mlir/test/Conversion/AMDGPUToROCDL/mfma-fp8-invalid.mlir
new file mode 100644
index 00000000000000..18d4f2663e40c8
--- /dev/null
+++ b/mlir/test/Conversion/AMDGPUToROCDL/mfma-fp8-invalid.mlir
@@ -0,0 +1,21 @@
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.08-amd-amdhsa --split-input-file --verify-diagnostics
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.0a-amd-amdhsa --split-input-file --verify-diagnostics
+
+// gfx908 and gfx90a have FeatureMAIInsts but no fp8 conversions at all, so the
+// fp8 MFMAs -- which first appear on gfx942 -- must not be selected for them.
+
+func.func @mfma_bf8(%arg0 : vector<8xf8E5M2FNUZ>, %arg1 : vector<4xf32>) {
+ // expected-error at below {{op no intrinsic matching MFMA size on given chipset}}
+ // expected-error at below {{failed to legalize operation 'amdgpu.mfma'}}
+ amdgpu.mfma 16x16x32 %arg0 * %arg0 + %arg1 : vector<8xf8E5M2FNUZ>, vector<8xf8E5M2FNUZ>, vector<4xf32>
+ func.return
+}
+
+// -----
+
+func.func @mfma_fp8(%arg0 : vector<8xf8E4M3FNUZ>, %arg1 : vector<4xf32>) {
+ // expected-error at below {{op no intrinsic matching MFMA size on given chipset}}
+ // expected-error at below {{failed to legalize operation 'amdgpu.mfma'}}
+ amdgpu.mfma 16x16x32 %arg0 * %arg0 + %arg1 : vector<8xf8E4M3FNUZ>, vector<8xf8E4M3FNUZ>, vector<4xf32>
+ func.return
+}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir b/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir
index ba5e096a642f62..8e5bd123121bec 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx950 -cse | FileCheck %s
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.50-amd-amdhsa -cse | FileCheck %s
func.func @mfma_to_rocdl(%arg0 : vector<8xf16>, %arg1 : vector<16xf32>,
%arg2 : vector<4xf32>, %arg3 : vector<8xbf16>,
%arg4 : vector<16xi8>, %arg5 : vector<16xi32>,
@@ -96,14 +96,3 @@ func.func @scaled_mfma_to_rocdl(%arg0 : vector<16xf32>,
func.return
}
-
-// CHECK-LABEL: func @mfma_reduce_precision_to_rocdl
-func.func @mfma_reduce_precision_to_rocdl(%arg0 : vector<2xf32>,
- %arg1 : vector<16xf32>,
- %arg2 : vector<4xf32>) {
- // CHECK: rocdl.mfma.f32.32x32x4.xf32
- amdgpu.mfma 32x32x4 %arg0 * %arg0 + %arg1 reducePrecision : vector<2xf32>, vector<2xf32>, vector<16xf32>
- // CHECK: rocdl.mfma.f32.16x16x8.xf32
- amdgpu.mfma 16x16x8 %arg0 * %arg0 + %arg2 reducePrecision : vector<2xf32>, vector<2xf32>, vector<4xf32>
- func.return
-}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/mfma-reduce-precision-invalid.mlir b/mlir/test/Conversion/AMDGPUToROCDL/mfma-reduce-precision-invalid.mlir
new file mode 100644
index 00000000000000..87e6cd86f869cc
--- /dev/null
+++ b/mlir/test/Conversion/AMDGPUToROCDL/mfma-reduce-precision-invalid.mlir
@@ -0,0 +1,23 @@
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.50-amd-amdhsa --split-input-file --verify-diagnostics
+
+// The xf32 MFMAs come from FeatureXF32Insts, which only gfx942 has. gfx950
+// compares greater than gfx942 by ISA version, so a version-ordered check let
+// them through here.
+
+func.func @mfma_reduce_precision_32x32x4(%arg0 : vector<2xf32>,
+ %arg1 : vector<16xf32>) {
+ // expected-error at below {{op no intrinsic matching MFMA size on given chipset}}
+ // expected-error at below {{failed to legalize operation 'amdgpu.mfma'}}
+ amdgpu.mfma 32x32x4 %arg0 * %arg0 + %arg1 reducePrecision : vector<2xf32>, vector<2xf32>, vector<16xf32>
+ func.return
+}
+
+// -----
+
+func.func @mfma_reduce_precision_16x16x8(%arg0 : vector<2xf32>,
+ %arg1 : vector<4xf32>) {
+ // expected-error at below {{op no intrinsic matching MFMA size on given chipset}}
+ // expected-error at below {{failed to legalize operation 'amdgpu.mfma'}}
+ amdgpu.mfma 16x16x8 %arg0 * %arg0 + %arg1 reducePrecision : vector<2xf32>, vector<2xf32>, vector<4xf32>
+ func.return
+}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir b/mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir
index 464d2d20048a27..2440851fc099f7 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx942 -cse | FileCheck %s
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa -cse | FileCheck %s
func.func @mfma_to_rocdl(%arg0 : f32, %arg1 : vector<32xf32>,
%arg2 : vector<16xf32>, %arg3 : vector<4xf32>,
%arg4 : vector<4xf16>, %arg5 : vector<4xi8>,
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/packed-ext.mlir b/mlir/test/Conversion/AMDGPUToROCDL/packed-ext.mlir
index ad2e7684afc4a3..3ba186c8099ae1 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/packed-ext.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/packed-ext.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx950 | FileCheck %s
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.50-amd-amdhsa | FileCheck %s
// CHECK-LABEL: func.func @scaled_ext_full_f8e4m3_f32
// CHECK-DAG: [[CAST:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<4xf8E4M3FN> to vector<4xi8>
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/packed-trunc-invalid.mlir b/mlir/test/Conversion/AMDGPUToROCDL/packed-trunc-invalid.mlir
index 93bfc60ce8f4be..6837882d7aa9e1 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/packed-trunc-invalid.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/packed-trunc-invalid.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx942 --split-input-file --verify-diagnostics
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa --split-input-file --verify-diagnostics
func.func @packed_trunc_ocp_type_requires_ocp_chipset(%arg0: f32) {
// expected-error at below {{'amdgpu.packed_trunc_2xfp8' op no truncation to result type available on given chipset}}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/packed-trunc.mlir b/mlir/test/Conversion/AMDGPUToROCDL/packed-trunc.mlir
index e9764d34cefaf8..e78f6ac8891ee5 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/packed-trunc.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/packed-trunc.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx950 | FileCheck %s
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.50-amd-amdhsa | FileCheck %s
// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_f32
// CHECK-DAG: [[ZERO:%.+]] = llvm.mlir.zero : vector<2xi16>
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/permlane-gfx1200-invalid.mlir b/mlir/test/Conversion/AMDGPUToROCDL/permlane-gfx1200-invalid.mlir
new file mode 100644
index 00000000000000..c9745d7fbe08d0
--- /dev/null
+++ b/mlir/test/Conversion/AMDGPUToROCDL/permlane-gfx1200-invalid.mlir
@@ -0,0 +1,21 @@
+// RUN: mlir-opt --convert-amdgpu-to-rocdl=arch=amdgpu12.00-amd-amdhsa --split-input-file --verify-diagnostics %s
+
+// gfx1200 has neither FeaturePermlane16Swap nor FeaturePermlane32Swap, but
+// compares greater than gfx950 by ISA version, so a version-ordered check
+// accepted both widths.
+
+func.func @permlane16(%arg0 : i32) -> i32 {
+ // expected-error at below {{op permlane_swap of row length 16 is not supported}}
+ // expected-error at below {{failed to legalize operation 'amdgpu.permlane_swap'}}
+ %0 = amdgpu.permlane_swap %arg0 16 : i32
+ return %0 : i32
+}
+
+// -----
+
+func.func @permlane32(%arg0 : i32) -> i32 {
+ // expected-error at below {{op permlane_swap of row length 32 is not supported}}
+ // expected-error at below {{failed to legalize operation 'amdgpu.permlane_swap'}}
+ %0 = amdgpu.permlane_swap %arg0 32 : i32
+ return %0 : i32
+}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/permlane-gfx1250-invalid.mlir b/mlir/test/Conversion/AMDGPUToROCDL/permlane-gfx1250-invalid.mlir
new file mode 100644
index 00000000000000..5280aaa90305a1
--- /dev/null
+++ b/mlir/test/Conversion/AMDGPUToROCDL/permlane-gfx1250-invalid.mlir
@@ -0,0 +1,11 @@
+// RUN: mlir-opt --convert-amdgpu-to-rocdl=arch=amdgpu12.50-amd-amdhsa --split-input-file --verify-diagnostics %s
+
+// gfx1250 has FeaturePermlane16Swap but not FeaturePermlane32Swap; the 16-wide
+// form is covered as a positive case in permlane.mlir.
+
+func.func @permlane32(%arg0 : i32) -> i32 {
+ // expected-error at below {{op permlane_swap of row length 32 is not supported}}
+ // expected-error at below {{failed to legalize operation 'amdgpu.permlane_swap'}}
+ %0 = amdgpu.permlane_swap %arg0 32 : i32
+ return %0 : i32
+}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/permlane-gfx1250.mlir b/mlir/test/Conversion/AMDGPUToROCDL/permlane-gfx1250.mlir
new file mode 100644
index 00000000000000..a6b76ce39498ba
--- /dev/null
+++ b/mlir/test/Conversion/AMDGPUToROCDL/permlane-gfx1250.mlir
@@ -0,0 +1,12 @@
+// RUN: mlir-opt --convert-amdgpu-to-rocdl=arch=amdgpu12.50-amd-amdhsa --canonicalize %s | FileCheck %s
+
+// gfx1250 has FeaturePermlane16Swap. It does not have FeaturePermlane32Swap;
+// see permlane-gfx1250-invalid.mlir.
+
+// CHECK-LABEL: func @permlane16_i32
+// CHECK-SAME: (%[[ARG0:.*]]: i32)
+func.func @permlane16_i32(%arg0 : i32) -> i32 {
+// CHECK: %[[PERM:.*]] = rocdl.permlane16.swap %[[ARG0]], %[[ARG0]], false, false : (i32, i32) -> <(i32, i32)>
+ %0 = amdgpu.permlane_swap %arg0 16 : i32
+ return %0 : i32
+}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/permlane-var.mlir b/mlir/test/Conversion/AMDGPUToROCDL/permlane-var.mlir
index e6e5a3061be5b0..3ed10ff574bfa4 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/permlane-var.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/permlane-var.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt --convert-amdgpu-to-rocdl=chipset=gfx1200 --canonicalize %s | FileCheck %s
+// RUN: mlir-opt --convert-amdgpu-to-rocdl=arch=amdgpu12.00-amd-amdhsa --canonicalize %s | FileCheck %s
// CHECK-LABEL: func @test_permlane_var_i32
// CHECK-SAME: (%[[SRC:.*]]: i32, %[[SEL:.*]]: i32)
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/permlane.mlir b/mlir/test/Conversion/AMDGPUToROCDL/permlane.mlir
index 9fc49dfeeb9fb5..208d82b0d068c1 100755
--- a/mlir/test/Conversion/AMDGPUToROCDL/permlane.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/permlane.mlir
@@ -1,5 +1,4 @@
-// RUN: mlir-opt --convert-amdgpu-to-rocdl=chipset=gfx950 --canonicalize %s | FileCheck %s
-// RUN: mlir-opt --convert-amdgpu-to-rocdl=chipset=gfx1200 --canonicalize %s | FileCheck %s
+// RUN: mlir-opt --convert-amdgpu-to-rocdl=arch=amdgpu9.50-amd-amdhsa --canonicalize %s | FileCheck %s
// CHECK-LABEL: func @test_permlane16_i32
// CHECK-SAME: (%[[ARG0:.*]]: i32)
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/sparse-mfma-gfx950.mlir b/mlir/test/Conversion/AMDGPUToROCDL/sparse-mfma-gfx950.mlir
index abdfba9689c8fa..3c3efee6546d8c 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/sparse-mfma-gfx950.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/sparse-mfma-gfx950.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx950 -cse | FileCheck %s
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.50-amd-amdhsa -cse | FileCheck %s
func.func @sparse_mfma_to_rocdl(%arg0 : vector<8xf16>, %arg1 : vector<16xf16>,
%arg2 : vector<4xf32>, %arg3 : vector<16xf32>,
%arg4 : vector<8xbf16>, %arg5 : vector<16xbf16>,
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/sparse-mfma.mlir b/mlir/test/Conversion/AMDGPUToROCDL/sparse-mfma.mlir
index 304f90351faf8b..3f49e9dcc80f22 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/sparse-mfma.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/sparse-mfma.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx942 -cse | FileCheck %s
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa -cse | FileCheck %s
func.func @sparse_mfma_to_rocdl(%arg0 : vector<4xf16>, %arg1 : vector<8xf16>,
%arg2 : vector<4xf32>, %arg3 : vector<16xf32>,
%arg4 : vector<4xbf16>, %arg5 : vector<8xbf16>,
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/swizzle.mlir b/mlir/test/Conversion/AMDGPUToROCDL/swizzle.mlir
index ef439efde1bd02..f04e95c92a0781 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/swizzle.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/swizzle.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-amdgpu-to-rocdl --canonicalize %s | FileCheck %s
+// RUN: mlir-opt -convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa --canonicalize %s | FileCheck %s
// CHECK-LABEL: func @test_swizzle_i32
// CHECK-SAME: (%[[ARG0:.*]]: i32)
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/swmmac-gfx12.mlir b/mlir/test/Conversion/AMDGPUToROCDL/swmmac-gfx12.mlir
index 61d533b75907d4..05bbe874e3faa3 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/swmmac-gfx12.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/swmmac-gfx12.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1200 --split-input-file --verify-diagnostics | FileCheck %s
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu12.00-amd-amdhsa --split-input-file --verify-diagnostics | FileCheck %s
// CHECK-LABEL: @rocdl.swmmac
func.func @rocdl.swmmac(
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/swmmac-gfx1250.mlir b/mlir/test/Conversion/AMDGPUToROCDL/swmmac-gfx1250.mlir
index 155e36c369a88b..5aae16be1821f6 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/swmmac-gfx1250.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/swmmac-gfx1250.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1250 --split-input-file --verify-diagnostics | FileCheck %s
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu12.50-amd-amdhsa --split-input-file --verify-diagnostics | FileCheck %s
// CHECK-LABEL: @rocdl.swmmac
func.func @rocdl.swmmac(
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/transpose_load.mlir b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load.mlir
index dcc6624cdb37b8..3f1074ebd6b571 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/transpose_load.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=chipset=gfx950 | FileCheck %s
-// RUN: not mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=chipset=gfx942 2>&1 | FileCheck %s --check-prefix=CHECK-OLD
+// RUN: mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=arch=amdgpu9.50-amd-amdhsa | FileCheck %s
+// RUN: not mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa 2>&1 | FileCheck %s --check-prefix=CHECK-OLD
// CHECK-LABEL: func @transpose_load_to_rocdl_4xf16
func.func @transpose_load_to_rocdl_4xf16(%idx1 : index, %idx2 : index, %wgmem : memref<128x72xf16, 3>) -> vector<4xf16> {
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_gfx1250.mlir b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_gfx1250.mlir
index 98ce6b7ea3001e..79ae69eeb2046f 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_gfx1250.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_gfx1250.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=chipset=gfx1250 | FileCheck %s
+// RUN: mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=arch=amdgpu12.50-amd-amdhsa | FileCheck %s
// CHECK-LABEL: func @transpose_load_to_rocdl_8xf16
func.func @transpose_load_to_rocdl_8xf16(%idx1 : index, %idx2 : index, %wgmem : memref<128x72xf16, 3>) -> vector<8xf16> {
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_gfx1250_invalid.mlir b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_gfx1250_invalid.mlir
index 61acdfe245c7cb..7cec509a5aef89 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_gfx1250_invalid.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_gfx1250_invalid.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --split-input-file --verify-diagnostics -convert-amdgpu-to-rocdl=chipset=gfx1250
+// RUN: mlir-opt %s --split-input-file --verify-diagnostics -convert-amdgpu-to-rocdl=arch=amdgpu12.50-amd-amdhsa
func.func @transpose_load_to_rocdl_4xf16(%idx1 : index, %idx2 : index, %wgmem : memref<128x72xf16, 3>) -> vector<4xf16> {
// expected-error at +2 {{'amdgpu.transpose_load' op 16-bit transpose_load requires 8 elements on gfx1250+}}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_gfx950_invalid.mlir b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_gfx950_invalid.mlir
index 682f989ede83be..ad685f5807c041 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_gfx950_invalid.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_gfx950_invalid.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --split-input-file --verify-diagnostics -convert-amdgpu-to-rocdl=chipset=gfx950
+// RUN: mlir-opt %s --split-input-file --verify-diagnostics -convert-amdgpu-to-rocdl=arch=amdgpu9.50-amd-amdhsa
func.func @transpose_load_to_rocdl_8xf16(%idx1 : index, %idx2 : index, %wgmem : memref<128x72xf16, 3>) -> vector<8xf16> {
// expected-error at +2 {{'amdgpu.transpose_load' op 16-bit transpose_load requires 4 elements on gfx950}}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_reject.mlir b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_reject.mlir
index a41051c904ed8d..08698ca12788e9 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_reject.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_reject.mlir
@@ -1,4 +1,4 @@
-// RUN: not mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=chipset=gfx950 2>&1 | FileCheck %s
+// RUN: not mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=arch=amdgpu9.50-amd-amdhsa 2>&1 | FileCheck %s
// -----
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx11.mlir b/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx11.mlir
index 08fd68dfe158de..f25b74d6603879 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx11.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx11.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1100 | FileCheck %s
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu11.00-amd-amdhsa | FileCheck %s
// CHECK-LABEL: @wmma_to_rocdl
func.func @wmma_to_rocdl(%arg0 : vector<16xf16>, %arg1 : vector<8xf32>, %arg2 : vector<4xf32>,
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx12.mlir b/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx12.mlir
index 1dac83946fc4cf..dca4df3351d8aa 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx12.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx12.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1200 \
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu12.00-amd-amdhsa \
// RUN: --split-input-file --verify-diagnostics | FileCheck %s
// CHECK-LABEL: @wmma_to_rocdl
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx1250.mlir b/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx1250.mlir
index 7f9605ad1a7eb9..23be3f33e7aee7 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx1250.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx1250.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=chipset=gfx1250 \
+// RUN: mlir-opt %s --convert-amdgpu-to-rocdl=arch=amdgpu12.50-amd-amdhsa \
// RUN: --split-input-file --verify-diagnostics | FileCheck %s
// CHECK-LABEL: @wmma_k4
diff --git a/mlir/test/Conversion/ArithToAMDGPU/16-bit-floats.mlir b/mlir/test/Conversion/ArithToAMDGPU/16-bit-floats.mlir
index 6077ef349408f9..273f373a9f614a 100644
--- a/mlir/test/Conversion/ArithToAMDGPU/16-bit-floats.mlir
+++ b/mlir/test/Conversion/ArithToAMDGPU/16-bit-floats.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="allow-packed-f16-round-to-zero=true" | FileCheck %s
+// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="arch=amdgpu9.42-amd-amdhsa allow-packed-f16-round-to-zero=true" | FileCheck %s
// CHECK-LABEL: @scalar_trunc
// CHECK-SAME: (%[[value:.*]]: f32)
diff --git a/mlir/test/Conversion/ArithToAMDGPU/8-bit-float-saturation-ocp.mlir b/mlir/test/Conversion/ArithToAMDGPU/8-bit-float-saturation-ocp.mlir
index e3c2ae95159395..e4d4aebf87241b 100644
--- a/mlir/test/Conversion/ArithToAMDGPU/8-bit-float-saturation-ocp.mlir
+++ b/mlir/test/Conversion/ArithToAMDGPU/8-bit-float-saturation-ocp.mlir
@@ -1,9 +1,9 @@
// RUN: mlir-opt --split-input-file %s \
-// RUN: --pass-pipeline='builtin.module(func.func(convert-arith-to-amdgpu{chipset=gfx950 saturate-fp8-truncf=true}))' \
+// RUN: --pass-pipeline='builtin.module(func.func(convert-arith-to-amdgpu{arch=amdgpu9.50-amd-amdhsa saturate-fp8-truncf=true}))' \
// RUN: | FileCheck %s
// RUN: mlir-opt --split-input-file %s \
-// RUN: --pass-pipeline='builtin.module(func.func(convert-arith-to-amdgpu{chipset=gfx1200 saturate-fp8-truncf=true}))' \
+// RUN: --pass-pipeline='builtin.module(func.func(convert-arith-to-amdgpu{arch=amdgpu12.00-amd-amdhsa saturate-fp8-truncf=true}))' \
// RUN: | FileCheck %s
// CHECK-LABEL: func.func @scalar_trunc
diff --git a/mlir/test/Conversion/ArithToAMDGPU/8-bit-float-saturation.mlir b/mlir/test/Conversion/ArithToAMDGPU/8-bit-float-saturation.mlir
index b6eabe391c0cd7..26a0ed993c2fac 100644
--- a/mlir/test/Conversion/ArithToAMDGPU/8-bit-float-saturation.mlir
+++ b/mlir/test/Conversion/ArithToAMDGPU/8-bit-float-saturation.mlir
@@ -1,5 +1,5 @@
// RUN: mlir-opt --split-input-file %s \
-// RUN: --pass-pipeline='builtin.module(func.func(convert-arith-to-amdgpu{chipset=gfx942 saturate-fp8-truncf=true}))' \
+// RUN: --pass-pipeline='builtin.module(func.func(convert-arith-to-amdgpu{arch=amdgpu9.42-amd-amdhsa saturate-fp8-truncf=true}))' \
// RUN: | FileCheck %s
// CHECK-LABEL: func.func @scalar_trunc
diff --git a/mlir/test/Conversion/ArithToAMDGPU/8-bit-floats-ocp.mlir b/mlir/test/Conversion/ArithToAMDGPU/8-bit-floats-ocp.mlir
index 91a9a57898761c..a9e2ae202ce735 100644
--- a/mlir/test/Conversion/ArithToAMDGPU/8-bit-floats-ocp.mlir
+++ b/mlir/test/Conversion/ArithToAMDGPU/8-bit-floats-ocp.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="chipset=gfx950" | FileCheck %s
-// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="chipset=gfx1200" | FileCheck %s
-
+// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="arch=amdgpu9.50-amd-amdhsa" | FileCheck %s
+// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="arch=amdgpu12.00-amd-amdhsa" | FileCheck %s
+
// CHECK-LABEL: func.func @scalar_ext
// CHECK-SAME: ([[V:%.+]]: f8E5M2)
// CHECK: [[FLOAT:%.+]] = amdgpu.ext_packed_fp8 [[V]][0] : f8E5M2 to f32
diff --git a/mlir/test/Conversion/ArithToAMDGPU/8-bit-floats.mlir b/mlir/test/Conversion/ArithToAMDGPU/8-bit-floats.mlir
index cf3133cf09add9..b8f2686f721409 100644
--- a/mlir/test/Conversion/ArithToAMDGPU/8-bit-floats.mlir
+++ b/mlir/test/Conversion/ArithToAMDGPU/8-bit-floats.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="chipset=gfx942" | FileCheck %s
+// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="arch=amdgpu9.42-amd-amdhsa" | FileCheck %s
// CHECK-LABEL: func.func @scalar_ext
// CHECK-SAME: ([[V:%.+]]: f8E5M2FNUZ)
diff --git a/mlir/test/Conversion/ArithToAMDGPU/deprecated-chipset-alias.mlir b/mlir/test/Conversion/ArithToAMDGPU/deprecated-chipset-alias.mlir
new file mode 100644
index 00000000000000..cb72ea0525e741
--- /dev/null
+++ b/mlir/test/Conversion/ArithToAMDGPU/deprecated-chipset-alias.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-opt %s -convert-arith-to-amdgpu='arch=gfx1030 chipset=gfx942' \
+// RUN: | FileCheck %s --check-prefix=ARCH-WINS
+// RUN: mlir-opt %s -convert-arith-to-amdgpu='chipset=gfx942' \
+// RUN: | FileCheck %s --check-prefix=ALIAS
+
+// Errors name whichever option supplied the target, so a stale `chipset` is
+// reported by its own value rather than by `arch`'s unusable default.
+// RUN: not mlir-opt %s -convert-arith-to-amdgpu='chipset=gfx999' 2>&1 \
+// RUN: | FileCheck %s --check-prefix=BAD-ALIAS
+// BAD-ALIAS: 'gfx999' is not a valid AMDGPU architecture
+
+// With neither given, the unusable default is still what gets reported.
+// RUN: not mlir-opt %s -convert-arith-to-amdgpu 2>&1 \
+// RUN: | FileCheck %s --check-prefix=NEITHER
+// NEITHER: 'invalid' is not a valid AMDGPU architecture
+
+// ARCH-WINS-LABEL: func @truncf_to_fp8
+// ARCH-WINS: arith.truncf
+// ARCH-WINS-NOT: amdgpu.packed_trunc_2xfp8
+
+// ALIAS-LABEL: func @truncf_to_fp8
+// ALIAS: amdgpu.packed_trunc_2xfp8
+// ALIAS-NOT: arith.truncf
+func.func @truncf_to_fp8(%x: f32) -> f8E4M3FNUZ {
+ %r = arith.truncf %x : f32 to f8E4M3FNUZ
+ return %r : f8E4M3FNUZ
+}
diff --git a/mlir/test/Conversion/ArithToAMDGPU/scaling-extf.mlir b/mlir/test/Conversion/ArithToAMDGPU/scaling-extf.mlir
index fe5b0520e37c16..e17b5f2cf16c76 100644
--- a/mlir/test/Conversion/ArithToAMDGPU/scaling-extf.mlir
+++ b/mlir/test/Conversion/ArithToAMDGPU/scaling-extf.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="chipset=gfx950" | FileCheck %s
-// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="chipset=gfx1100" | FileCheck %s --check-prefix=CHECK-GFX1100
+// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="arch=amdgpu9.50-amd-amdhsa" | FileCheck %s
+// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="arch=amdgpu11.00-amd-amdhsa" | FileCheck %s --check-prefix=CHECK-GFX1100
// CHECK-LABEL: @conversion_f8_f32_fallback
// CHECK: %[[CST:.+]] = arith.constant dense<0.000000e+00> : vector<2x2xf32>
diff --git a/mlir/test/Conversion/ArithToAMDGPU/scaling-truncf-tensor.mlir b/mlir/test/Conversion/ArithToAMDGPU/scaling-truncf-tensor.mlir
index d22f35a6d07f1d..4d94f2a5efb743 100644
--- a/mlir/test/Conversion/ArithToAMDGPU/scaling-truncf-tensor.mlir
+++ b/mlir/test/Conversion/ArithToAMDGPU/scaling-truncf-tensor.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-arith-to-amdgpu=chipset=gfx950 | FileCheck %s
+// RUN: mlir-opt %s -convert-arith-to-amdgpu=arch=amdgpu9.50-amd-amdhsa | FileCheck %s
// CHECK-LABEL: func.func @m0
// CHECK: arith.scaling_truncf
diff --git a/mlir/test/Conversion/ArithToAMDGPU/scaling-truncf.mlir b/mlir/test/Conversion/ArithToAMDGPU/scaling-truncf.mlir
index 4c707680378158..cd26e12be49399 100644
--- a/mlir/test/Conversion/ArithToAMDGPU/scaling-truncf.mlir
+++ b/mlir/test/Conversion/ArithToAMDGPU/scaling-truncf.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="chipset=gfx950" | FileCheck %s
-// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="chipset=gfx1100" | FileCheck %s --check-prefix=CHECK-GFX1100
+// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="arch=amdgpu9.50-amd-amdhsa" | FileCheck %s
+// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="arch=amdgpu11.00-amd-amdhsa" | FileCheck %s --check-prefix=CHECK-GFX1100
// CHECK-LABEL: @conversion_f8_fallback
// CHECK-DAG: %[[CST:.+]] = arith.constant dense<0.000000e+00> : vector<2x2xf8E5M2>
diff --git a/mlir/test/Conversion/GPUCommon/lower-global-id.mlir b/mlir/test/Conversion/GPUCommon/lower-global-id.mlir
index 94b9f90052769e..18baa4e013f347 100644
--- a/mlir/test/Conversion/GPUCommon/lower-global-id.mlir
+++ b/mlir/test/Conversion/GPUCommon/lower-global-id.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -convert-gpu-to-rocdl | FileCheck %s --check-prefixes=ROCDL
+// RUN: mlir-opt %s -split-input-file -convert-gpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa | FileCheck %s --check-prefixes=ROCDL
// RUN: mlir-opt %s -split-input-file -convert-gpu-to-nvvm | FileCheck %s --check-prefixes=NVVM
gpu.module @kernel {
diff --git a/mlir/test/Conversion/GPUCommon/lower-memory-space-attrs.mlir b/mlir/test/Conversion/GPUCommon/lower-memory-space-attrs.mlir
index 3b794ab717c03f..9050122bb71785 100644
--- a/mlir/test/Conversion/GPUCommon/lower-memory-space-attrs.mlir
+++ b/mlir/test/Conversion/GPUCommon/lower-memory-space-attrs.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -convert-gpu-to-rocdl | FileCheck %s --check-prefixes=CHECK,ROCDL
+// RUN: mlir-opt %s -split-input-file -convert-gpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa | FileCheck %s --check-prefixes=CHECK,ROCDL
// RUN: mlir-opt %s -split-input-file -convert-gpu-to-nvvm | FileCheck %s --check-prefixes=CHECK,NVVM
gpu.module @kernel {
diff --git a/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir b/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir
index 38e73ea9179ac6..fb3dfd81474360 100644
--- a/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir
+++ b/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir
@@ -1,5 +1,5 @@
// RUN: mlir-opt -allow-unregistered-dialect --convert-gpu-to-nvvm --split-input-file %s | FileCheck --check-prefix=NVVM %s
-// RUN: mlir-opt -allow-unregistered-dialect --convert-gpu-to-rocdl --split-input-file %s | FileCheck --check-prefix=ROCDL %s
+// RUN: mlir-opt -allow-unregistered-dialect --convert-gpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa --split-input-file %s | FileCheck --check-prefix=ROCDL %s
gpu.module @kernel {
// NVVM-LABEL: llvm.func @private
diff --git a/mlir/test/Conversion/GPUCommon/memref-arg-attrs.mlir b/mlir/test/Conversion/GPUCommon/memref-arg-attrs.mlir
index e7c742067b4eb5..3646c43313e1f9 100644
--- a/mlir/test/Conversion/GPUCommon/memref-arg-attrs.mlir
+++ b/mlir/test/Conversion/GPUCommon/memref-arg-attrs.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -convert-gpu-to-rocdl='use-bare-ptr-memref-call-conv=0' | FileCheck %s --check-prefixes=CHECK,ROCDL
+// RUN: mlir-opt %s -split-input-file -convert-gpu-to-rocdl='arch=amdgpu9.42-amd-amdhsa use-bare-ptr-memref-call-conv=0' | FileCheck %s --check-prefixes=CHECK,ROCDL
// RUN: mlir-opt %s -split-input-file -convert-gpu-to-nvvm='use-bare-ptr-memref-call-conv=0' | FileCheck %s --check-prefixes=CHECK,NVVM
gpu.module @kernel {
diff --git a/mlir/test/Conversion/GPUCommon/memref-arg-noalias-attrs.mlir b/mlir/test/Conversion/GPUCommon/memref-arg-noalias-attrs.mlir
index 33cdc3348e5137..0c097fdaa29613 100644
--- a/mlir/test/Conversion/GPUCommon/memref-arg-noalias-attrs.mlir
+++ b/mlir/test/Conversion/GPUCommon/memref-arg-noalias-attrs.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -convert-gpu-to-rocdl='use-bare-ptr-memref-call-conv=1' | FileCheck %s --check-prefixes=CHECK,ROCDL
+// RUN: mlir-opt %s -split-input-file -convert-gpu-to-rocdl='arch=amdgpu9.42-amd-amdhsa use-bare-ptr-memref-call-conv=1' | FileCheck %s --check-prefixes=CHECK,ROCDL
// RUN: mlir-opt %s -split-input-file -convert-gpu-to-nvvm='use-bare-ptr-memref-call-conv=1' | FileCheck %s --check-prefixes=CHECK,NVVM
gpu.module @kernel {
diff --git a/mlir/test/Conversion/GPUCommon/memref-arg-noalias-warning.mlir b/mlir/test/Conversion/GPUCommon/memref-arg-noalias-warning.mlir
index 793df7380d78bd..233f21a28b59ab 100644
--- a/mlir/test/Conversion/GPUCommon/memref-arg-noalias-warning.mlir
+++ b/mlir/test/Conversion/GPUCommon/memref-arg-noalias-warning.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -convert-gpu-to-rocdl='use-bare-ptr-memref-call-conv=0' -verify-diagnostics
+// RUN: mlir-opt %s -split-input-file -convert-gpu-to-rocdl='arch=amdgpu9.42-amd-amdhsa use-bare-ptr-memref-call-conv=0' -verify-diagnostics
gpu.module @kernel {
// expected-warning @+1 {{Cannot copy noalias with non-bare pointers.}}
diff --git a/mlir/test/Conversion/GPUToROCDL/constant-address-space.mlir b/mlir/test/Conversion/GPUToROCDL/constant-address-space.mlir
index 738aece1769dab..9c5ab4bf2edaa9 100644
--- a/mlir/test/Conversion/GPUToROCDL/constant-address-space.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/constant-address-space.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-gpu-to-rocdl %s | FileCheck %s
+// RUN: mlir-opt -convert-gpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa %s | FileCheck %s
module attributes {gpu.container_module} {
gpu.module @kernel_module {
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-barrier.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-barrier.mlir
index 618d1889b84785..8da4709b672cd8 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-barrier.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-barrier.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx950' --mlir-print-local-scope | FileCheck %s --check-prefixes=CHECK,GFX9
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx1201' --mlir-print-local-scope | FileCheck %s --check-prefixes=CHECK,GFX12
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu9.50-amd-amdhsa' --mlir-print-local-scope | FileCheck %s --check-prefixes=CHECK,GFX9
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu12.01-amd-amdhsa' --mlir-print-local-scope | FileCheck %s --check-prefixes=CHECK,GFX12
gpu.module @test_module {
// CHECK-LABEL: func @barrier_default()
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-barriers-gfx12.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-barriers-gfx12.mlir
index 402dcae5e98323..ab4914f331d495 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-barriers-gfx12.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-barriers-gfx12.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx1250' --mlir-print-local-scope | FileCheck %s
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu12.50-amd-amdhsa' --mlir-print-local-scope | FileCheck %s
gpu.module @test_module {
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-hip.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-hip.mlir
index 32da31202b6884..d2317214d809a2 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-hip.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-hip.mlir
@@ -1,4 +1,6 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='runtime=HIP' -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu9.42-amd-amdhsa runtime=HIP' -split-input-file | FileCheck %s
+// Ensure old-style options work
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx942 runtime=HIP' -split-input-file | FileCheck %s
// CHECK-LABEL: gpu.module @test_module
gpu.module @test_module {
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-invalid-ballot.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-invalid-ballot.mlir
index a94ab3b5bb780b..ad199e413b9b9f 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-invalid-ballot.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-invalid-ballot.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx950' -split-input-file -verify-diagnostics
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu9.50-amd-amdhsa' -split-input-file -verify-diagnostics
// -----
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-invalid-dialect.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-invalid-dialect.mlir
index 117f7692669de8..d917746b12c2dc 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-invalid-dialect.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-invalid-dialect.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='allowed-dialects=test' -verify-diagnostics
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu9.42-amd-amdhsa allowed-dialects=test' -verify-diagnostics
// expected-error @+1 {{dialect does not implement ConvertToLLVMPatternInterface: test}}
gpu.module @test_module_1 {
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-invalid-named-barrier.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-invalid-named-barrier.mlir
index 3f39f4abcf396a..9a5750f0660ea9 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-invalid-named-barrier.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-invalid-named-barrier.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx1100' -split-input-file -verify-diagnostics
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu11.00-amd-amdhsa' -split-input-file -verify-diagnostics
gpu.module @test_module {
func.func @initialize_named_barrier_pre_gfx12(%count : i32) {
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-named-barrier-non-const.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-named-barrier-non-const.mlir
index c9ce2794f1422a..cd05533ef96542 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-named-barrier-non-const.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-named-barrier-non-const.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx1250' -split-input-file -verify-diagnostics
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu12.50-amd-amdhsa' -split-input-file -verify-diagnostics
gpu.module @test_module {
func.func @non_constant_member_count(%count : i32) {
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-opencl.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-opencl.mlir
index 00d1d7d8526809..6f71012a8afda5 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-opencl.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-opencl.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='runtime=OpenCL' | FileCheck %s
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu9.42-amd-amdhsa runtime=OpenCL' | FileCheck %s
gpu.module @test_module {
// CHECK: llvm.mlir.global internal constant @[[$PRINT_GLOBAL:[A-Za-z0-9_]+]]("Hello: %d\0A\00") {addr_space = 4 : i32}
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-subgroup-id.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-subgroup-id.mlir
index 9cab3ff48f5bf0..cc474fdaa999f6 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-subgroup-id.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-subgroup-id.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx942' | FileCheck %s --check-prefixes=CHECK,GFX9
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx1201' | FileCheck %s --check-prefixes=CHECK,GFX12
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu9.42-amd-amdhsa' | FileCheck %s --check-prefixes=CHECK,GFX9
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu12.01-amd-amdhsa' | FileCheck %s --check-prefixes=CHECK,GFX12
gpu.module @test_module {
// CHECK-LABEL: func @subgroup_id()
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
index c01af31e9d4f15..b4051ff32c5f21 100755
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx950' -split-input-file | FileCheck %s
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx950 allowed-dialects=func,arith,math' -split-input-file | FileCheck %s
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx950 index-bitwidth=32' -split-input-file | FileCheck --check-prefix=CHECK32 %s
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu9.50-amd-amdhsa' -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu9.50-amd-amdhsa allowed-dialects=func,arith,math' -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='arch=amdgpu9.50-amd-amdhsa index-bitwidth=32' -split-input-file | FileCheck --check-prefix=CHECK32 %s
// CHECK-LABEL: @test_module
// CHECK-SAME: llvm.data_layout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
diff --git a/mlir/test/Conversion/GPUToROCDL/memref.mlir b/mlir/test/Conversion/GPUToROCDL/memref.mlir
index e645481c892308..b6cc3150f003d2 100644
--- a/mlir/test/Conversion/GPUToROCDL/memref.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/memref.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -convert-gpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa -split-input-file | FileCheck %s
// RUN: mlir-opt %s \
-// RUN: -convert-gpu-to-rocdl='use-bare-ptr-memref-call-conv=true' \
+// RUN: -convert-gpu-to-rocdl='arch=amdgpu9.42-amd-amdhsa use-bare-ptr-memref-call-conv=true' \
// RUN: -split-input-file \
// RUN: | FileCheck %s --check-prefix=BARE
diff --git a/mlir/test/Conversion/GPUToROCDL/private-func-no-c-interface.mlir b/mlir/test/Conversion/GPUToROCDL/private-func-no-c-interface.mlir
index a9f778eae28209..b2975cc79404e2 100644
--- a/mlir/test/Conversion/GPUToROCDL/private-func-no-c-interface.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/private-func-no-c-interface.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -convert-gpu-to-rocdl=arch=amdgpu9.42-amd-amdhsa -split-input-file | FileCheck %s
// A private device function cannot be called from outside its module, so it
// must not be given a C interface wrapper; requesting one would only anchor
diff --git a/mlir/test/Conversion/MathToROCDL/math-to-rocdl.mlir b/mlir/test/Conversion/MathToROCDL/math-to-rocdl.mlir
index 455f8868396044..b25d30f5d9af87 100644
--- a/mlir/test/Conversion/MathToROCDL/math-to-rocdl.mlir
+++ b/mlir/test/Conversion/MathToROCDL/math-to-rocdl.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s -allow-unregistered-dialect -split-input-file -pass-pipeline='builtin.module(convert-math-to-rocdl{chipset=gfx803})' | FileCheck %s --check-prefix=PRE9
-// RUN: mlir-opt %s -allow-unregistered-dialect -split-input-file -pass-pipeline='builtin.module(convert-math-to-rocdl{chipset=gfx942})' | FileCheck %s --check-prefix=POST9
+// RUN: mlir-opt %s -allow-unregistered-dialect -split-input-file -pass-pipeline='builtin.module(convert-math-to-rocdl{arch=amdgpu8.03-amd-amdhsa})' | FileCheck %s --check-prefix=PRE9
+// RUN: mlir-opt %s -allow-unregistered-dialect -split-input-file -pass-pipeline='builtin.module(convert-math-to-rocdl{arch=amdgpu9.42-amd-amdhsa})' | FileCheck %s --check-prefix=POST9
module @test_module {
// CHECK: llvm.func @__ocml_fmod_f16(f16, f16) -> f16
diff --git a/mlir/test/Dialect/AMDGPU/amdgpu-emulate-atomics.mlir b/mlir/test/Dialect/AMDGPU/amdgpu-emulate-atomics.mlir
index fa883bb96c1a04..b4207d82ba2960 100644
--- a/mlir/test/Dialect/AMDGPU/amdgpu-emulate-atomics.mlir
+++ b/mlir/test/Dialect/AMDGPU/amdgpu-emulate-atomics.mlir
@@ -1,11 +1,11 @@
-// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx908 %s | FileCheck %s --check-prefixes=CHECK,GFX9CAS,GFX908
-// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx90a %s | FileCheck %s --check-prefixes=CHECK,GFX9CAS,GFX90A
-// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx90c %s | FileCheck %s --check-prefixes=CHECK,GFX9CAS,GFX90C
-// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx1030 %s | FileCheck %s --check-prefixes=CHECK,GFX10
-// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx1100 %s | FileCheck %s --check-prefixes=CHECK,GFX11
-// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx1200 %s | FileCheck %s --check-prefixes=CHECK,GFX12
-// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx942 %s | FileCheck %s --check-prefixes=CHECK,GFX9CAS,GFX942
-// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx950 %s | FileCheck %s --check-prefixes=CHECK,GFX9CAS,GFX950
+// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=arch=amdgpu9.08-amd-amdhsa %s | FileCheck %s --check-prefixes=CHECK,GFX9CAS,GFX9NOF64,GFX908
+// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=arch=amdgpu9.0a-amd-amdhsa %s | FileCheck %s --check-prefixes=CHECK,GFX9CAS,GFX90A
+// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=arch=amdgpu9.0c-amd-amdhsa %s | FileCheck %s --check-prefixes=CHECK,GFX9CAS,GFX9NOF64,GFX90C
+// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=arch=amdgpu10.30-amd-amdhsa %s | FileCheck %s --check-prefixes=CHECK,GFX10
+// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=arch=amdgpu11.00-amd-amdhsa %s | FileCheck %s --check-prefixes=CHECK,GFX11
+// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=arch=amdgpu12.00-amd-amdhsa %s | FileCheck %s --check-prefixes=CHECK,GFX12
+// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=arch=amdgpu9.42-amd-amdhsa %s | FileCheck %s --check-prefixes=CHECK,GFX9CAS,GFX942
+// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=arch=amdgpu9.50-amd-amdhsa %s | FileCheck %s --check-prefixes=CHECK,GFX9CAS,GFX950
// -----
@@ -46,19 +46,17 @@ func.func @atomic_fmax_f64(%val: f64, %buffer: memref<?xf64>, %idx: i32) {
// GFX12: amdgpu.raw_buffer_atomic_fmax boundsCheck(true) [[val]] -> [[buffer]][[[idx]]]
// GFX942: amdgpu.raw_buffer_atomic_fmax boundsCheck(true) [[val]] -> [[buffer]][[[idx]]]
// GFX950: amdgpu.raw_buffer_atomic_fmax boundsCheck(true) [[val]] -> [[buffer]][[[idx]]]
-// gfx908 has no f64 buffer fmin/fmax, so it is emulated.
-// GFX908: [[ld:%.+]] = amdgpu.raw_buffer_load boundsCheck(true) [[buffer]][[[idx]]]
-// GFX908: cf.br [[loop:\^.+]]([[ld]] : f64)
-// GFX908: [[loop]]([[arg:%.+]]: f64):
-// GFX908: [[operated:%.+]] = arith.maximumf [[val]], [[arg]]
-// GFX908: [[atomicRes:%.+]] = amdgpu.raw_buffer_atomic_cmpswap boundsCheck(true) [[operated]], [[arg]] -> [[buffer]][[[idx]]]
-// GFX908: [[argCast:%.+]] = arith.bitcast [[arg]] : f64 to i64
-// GFX908: [[resCast:%.+]] = arith.bitcast [[atomicRes]] : f64 to i64
-// GFX908: [[test:%.+]] = arith.cmpi eq, [[resCast]], [[argCast]]
-// GFX908: cf.cond_br [[test]], [[post:\^.+]]([[arg]] : f64), [[loop]]([[atomicRes]] : f64)
-// GFX908: [[post]]([[old:%.+]]: f64):
-// gfx90c has none either, but sorts after gfx90a by ISA version.
-// GFX90C: amdgpu.raw_buffer_atomic_fmax boundsCheck(true) [[val]] -> [[buffer]][[[idx]]]
+// Neither gfx908 nor gfx90c has f64 buffer fmin/fmax.
+// GFX9NOF64: [[ld:%.+]] = amdgpu.raw_buffer_load boundsCheck(true) [[buffer]][[[idx]]]
+// GFX9NOF64: cf.br [[loop:\^.+]]([[ld]] : f64)
+// GFX9NOF64: [[loop]]([[arg:%.+]]: f64):
+// GFX9NOF64: [[operated:%.+]] = arith.maximumf [[val]], [[arg]]
+// GFX9NOF64: [[atomicRes:%.+]] = amdgpu.raw_buffer_atomic_cmpswap boundsCheck(true) [[operated]], [[arg]] -> [[buffer]][[[idx]]]
+// GFX9NOF64: [[argCast:%.+]] = arith.bitcast [[arg]] : f64 to i64
+// GFX9NOF64: [[resCast:%.+]] = arith.bitcast [[atomicRes]] : f64 to i64
+// GFX9NOF64: [[test:%.+]] = arith.cmpi eq, [[resCast]], [[argCast]]
+// GFX9NOF64: cf.cond_br [[test]], [[post:\^.+]]([[arg]] : f64), [[loop]]([[atomicRes]] : f64)
+// GFX9NOF64: [[post]]([[old:%.+]]: f64):
// CHECK-NEXT: gpu.printf "End\0A"
gpu.printf "Begin\n"
%old = amdgpu.raw_buffer_atomic_fmax boundsCheck(true) %val -> %buffer[%idx] : f64 -> memref<?xf64>, i32
@@ -77,8 +75,11 @@ func.func @atomic_fadd(%val: f32, %buffer: memref<?xf32>, %idx: i32) {
// GFX12: amdgpu.raw_buffer_atomic_fadd
// GFX942: amdgpu.raw_buffer_atomic_fadd
// GFX950: amdgpu.raw_buffer_atomic_fadd
+// gfx908 only has the no-return form, which suffices here as %old is unused.
// GFX908: amdgpu.raw_buffer_atomic_fadd
-// GFX90C: amdgpu.raw_buffer_atomic_fadd
+// gfx90c has no buffer fadd at all.
+// GFX90C: amdgpu.raw_buffer_load
+// GFX90C: amdgpu.raw_buffer_atomic_cmpswap
%old = amdgpu.raw_buffer_atomic_fadd boundsCheck(true) %val -> %buffer[%idx] : f32 -> memref<?xf32>, i32
func.return
}
@@ -100,8 +101,11 @@ func.func @atomic_fadd_v2f16(%val: vector<2xf16>, %buffer: memref<?xf16>, %idx:
// GFX942: amdgpu.raw_buffer_atomic_fadd
// GFX12: amdgpu.raw_buffer_atomic_fadd
// GFX950: amdgpu.raw_buffer_atomic_fadd
-// GFX908: amdgpu.raw_buffer_atomic_fadd
-// GFX90C: amdgpu.raw_buffer_atomic_fadd
+// Neither gfx908 nor gfx90c has the packed f16 buffer fadd.
+// GFX908: amdgpu.raw_buffer_load
+// GFX908: amdgpu.raw_buffer_atomic_cmpswap
+// GFX90C: amdgpu.raw_buffer_load
+// GFX90C: amdgpu.raw_buffer_atomic_cmpswap
%old = amdgpu.raw_buffer_atomic_fadd boundsCheck(true) %val -> %buffer[%idx] : vector<2xf16> -> memref<?xf16>, i32
func.return
}
@@ -125,3 +129,24 @@ func.func @atomic_fadd_v2bf16(%val: vector<2xbf16>, %buffer: memref<?xbf16>, %id
%old = amdgpu.raw_buffer_atomic_fadd boundsCheck(true) %val -> %buffer[%idx] : vector<2xbf16> -> memref<?xbf16>, i32
func.return
}
+
+// -----
+
+// gfx908 has only the no-return buffer fadd, so a *used* result has to be
+// emulated even though the discarded-result case above lowers natively.
+// CHECK: func @atomic_fadd_used_result
+func.func @atomic_fadd_used_result(%val: f32, %buffer: memref<?xf32>, %idx: i32) -> f32 {
+// GFX908: amdgpu.raw_buffer_load
+// GFX908: amdgpu.raw_buffer_atomic_cmpswap
+// GFX90A: amdgpu.raw_buffer_atomic_fadd
+// GFX942: amdgpu.raw_buffer_atomic_fadd
+// GFX950: amdgpu.raw_buffer_atomic_fadd
+// GFX90C: amdgpu.raw_buffer_load
+// GFX90C: amdgpu.raw_buffer_atomic_cmpswap
+// GFX10: amdgpu.raw_buffer_load
+// GFX10: amdgpu.raw_buffer_atomic_cmpswap
+// GFX11: amdgpu.raw_buffer_atomic_fadd
+// GFX12: amdgpu.raw_buffer_atomic_fadd
+ %old = amdgpu.raw_buffer_atomic_fadd boundsCheck(true) %val -> %buffer[%idx] : f32 -> memref<?xf32>, i32
+ func.return %old : f32
+}
diff --git a/mlir/test/Dialect/GPU/promote-shuffle-amdgpu-invalid.mlir b/mlir/test/Dialect/GPU/promote-shuffle-amdgpu-invalid.mlir
new file mode 100644
index 00000000000000..46a9821cd6fcc5
--- /dev/null
+++ b/mlir/test/Dialect/GPU/promote-shuffle-amdgpu-invalid.mlir
@@ -0,0 +1,58 @@
+// RUN: mlir-opt %s --transform-interpreter --split-input-file --verify-diagnostics
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
+ %func = transform.structured.match ops{["func.func"]} in %module_op : (!transform.any_op) -> !transform.any_op
+ transform.apply_patterns to %func {
+ // expected-error at below {{'gfx999' is not a valid AMDGPU architecture}}
+ transform.apply_patterns.gpu.gpu_shuffle_to_amdgpu <arch = "gfx999">
+ } : !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Only xnack and sramecc are target-ID features, and only on a GPU that
+// supports switching them.
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
+ %func = transform.structured.match ops{["func.func"]} in %module_op : (!transform.any_op) -> !transform.any_op
+ transform.apply_patterns to %func {
+ // expected-error at below {{'gfx600:xnack+' is not a valid AMDGPU architecture}}
+ transform.apply_patterns.gpu.gpu_shuffle_to_amdgpu <arch = "gfx600:xnack+">
+ } : !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// A modifier without a +/- sign is not a target ID.
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
+ %func = transform.structured.match ops{["func.func"]} in %module_op : (!transform.any_op) -> !transform.any_op
+ transform.apply_patterns to %func {
+ // expected-error at below {{'gfx908:xnack' is not a valid AMDGPU architecture}}
+ transform.apply_patterns.gpu.gpu_shuffle_to_amdgpu <arch = "gfx908:xnack">
+ } : !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Wavefront size is not a target-ID feature, so it cannot ride on `arch`.
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
+ %func = transform.structured.match ops{["func.func"]} in %module_op : (!transform.any_op) -> !transform.any_op
+ transform.apply_patterns to %func {
+ // expected-error at below {{'gfx1030:wavefrontsize64+' is not a valid AMDGPU architecture}}
+ transform.apply_patterns.gpu.gpu_shuffle_to_amdgpu <arch = "gfx1030:wavefrontsize64+">
+ } : !transform.any_op
+ transform.yield
+ }
+}
diff --git a/mlir/test/Dialect/GPU/promote-shuffle-amdgpu.mlir b/mlir/test/Dialect/GPU/promote-shuffle-amdgpu.mlir
index 747c997a3b441b..d833d8ad58e445 100644
--- a/mlir/test/Dialect/GPU/promote-shuffle-amdgpu.mlir
+++ b/mlir/test/Dialect/GPU/promote-shuffle-amdgpu.mlir
@@ -4,7 +4,7 @@ module attributes {transform.with_named_sequence} {
transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
%func = transform.structured.match ops{["func.func"]} in %module_op : (!transform.any_op) -> !transform.any_op
transform.apply_patterns to %func {
- transform.apply_patterns.gpu.gpu_shuffle_to_amdgpu chipset = "gfx950"
+ transform.apply_patterns.gpu.gpu_shuffle_to_amdgpu <arch = "amdgpu9.50-amd-amdhsa">
} : !transform.any_op
transform.yield
}
diff --git a/mlir/test/Dialect/LLVMIR/attach-targets.mlir b/mlir/test/Dialect/LLVMIR/attach-targets.mlir
index eabea4c2bdab54..db507fb15091a9 100644
--- a/mlir/test/Dialect/LLVMIR/attach-targets.mlir
+++ b/mlir/test/Dialect/LLVMIR/attach-targets.mlir
@@ -5,7 +5,7 @@
// RUN: | FileCheck %s
// RUN: mlir-opt %s \
// RUN: --nvvm-attach-target='module=options.* O=1 chip=sm_70 fast=true ftz=true' \
-// RUN: --rocdl-attach-target='module=options.* l=file1.bc,file2.bc wave64=false finite-only=true' \
+// RUN: --rocdl-attach-target='module=options.* l=file1.bc,file2.bc wavesize=32 finite-only=true' \
// RUN: --xevm-attach-target='module=options.* O=1 chip=pvc' \
// RUN: | FileCheck %s --check-prefix=CHECK-OPTIONS
diff --git a/mlir/test/Dialect/LLVMIR/rocdl-attach-target-arch.mlir b/mlir/test/Dialect/LLVMIR/rocdl-attach-target-arch.mlir
new file mode 100644
index 00000000000000..b4a1402fdb5878
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/rocdl-attach-target-arch.mlir
@@ -0,0 +1,105 @@
+// `arch` is an alternative spelling for triple + chip. It is split apart
+// because the attribute feeds the TargetMachine, whose -mcpu only accepts a
+// bare processor name, and the triple is normalized to the new-style spelling
+// that names the GPU's subarch.
+
+// Every spelling of gfx90a lands on the same attribute, whichever triple it
+// arrived with.
+// RUN: mlir-opt %s --rocdl-attach-target='arch=gfx90a' \
+// RUN: | FileCheck %s
+// RUN: mlir-opt %s --rocdl-attach-target='arch=amdgcn-amd-amdhsa--gfx90a' \
+// RUN: | FileCheck %s
+// RUN: mlir-opt %s --rocdl-attach-target='arch=amdgpu9.0a-amd-amdhsa--gfx90a' \
+// RUN: | FileCheck %s
+
+// The legacy split options still attach what they always did, so migrating is
+// opt-in: only `arch` normalizes the triple or consults the target for the
+// wavefront size.
+// RUN: mlir-opt %s \
+// RUN: --rocdl-attach-target='triple=amdgcn-amd-amdhsa chip=gfx90a' \
+// RUN: | FileCheck %s --check-prefix=LEGACY
+
+// An explicit `features` is passed through untouched.
+// RUN: mlir-opt %s --rocdl-attach-target='arch=gfx90a features=+dpp' \
+// RUN: | FileCheck %s --check-prefix=FEATURES
+
+// The processor is the more specific fact, so a family triple normalizes down
+// to the subarch of the GPU it resolved to.
+// RUN: mlir-opt %s --rocdl-attach-target='arch=amdgpu9.4-amd-amdhsa--gfx950' \
+// RUN: | FileCheck %s --check-prefix=FAMILY
+
+// A generic target normalizes to its family subarch.
+// RUN: mlir-opt %s --rocdl-attach-target='arch=gfx9-4-generic' \
+// RUN: | FileCheck %s --check-prefix=GENERIC
+
+// The wavefront size comes from the target, so a wave32-only chip gets
+// no_wave64 rather than the `wave64` option's default.
+// RUN: mlir-opt %s --rocdl-attach-target='arch=gfx1250' \
+// RUN: | FileCheck %s --check-prefix=WAVE32
+
+// A dual-mode chip defaults to wave32 and honours an explicit `wavesize`. The
+// size lands in both the flags (for the device libraries) and the features (for
+// the TargetMachine), which must agree.
+// RUN: mlir-opt %s --rocdl-attach-target='arch=gfx1030' \
+// RUN: | FileCheck %s --check-prefix=WAVE32-RDNA
+// RUN: mlir-opt %s --rocdl-attach-target='arch=gfx1030 wavesize=64' \
+// RUN: | FileCheck %s --check-prefix=WAVE64-RDNA
+
+// Errors have no source location, so they are matched on stderr.
+// RUN: not mlir-opt %s --rocdl-attach-target='arch=gfx999' 2>&1 \
+// RUN: | FileCheck %s --check-prefix=ERR
+// ERR: 'gfx999' is not a valid AMDGPU architecture
+
+// A triple naming no GPU cannot be attached: the attribute requires a chip.
+// RUN: not mlir-opt %s --rocdl-attach-target='arch=amdgcn-amd-amdhsa' 2>&1 \
+// RUN: | FileCheck %s --check-prefix=NOGPU
+// NOGPU: 'amdgcn-amd-amdhsa' names no GPU
+
+// xnack/sramecc are module flags in the backend, not subtarget features, so
+// #rocdl.target has nowhere to carry them and the target ID's modifiers land on
+// the module instead. A modifier the target ID omits stays omitted, since an
+// absent flag means "either" and false would be a different request.
+// RUN: mlir-opt %s --rocdl-attach-target='arch=gfx90a:xnack+' \
+// RUN: | FileCheck %s --check-prefix=XNACK
+// RUN: mlir-opt %s --rocdl-attach-target='arch=gfx90a:sramecc-:xnack-' \
+// RUN: | FileCheck %s --check-prefix=BOTH
+
+module attributes {gpu.container_module} {
+
+// CHECK-LABEL: @rocdl_module
+// CHECK-SAME: [#rocdl.target<triple = "amdgpu9.0a-amd-amdhsa", chip = "gfx90a">]
+// CHECK-NOT: rocdl.xnack
+// CHECK-NOT: rocdl.sramecc
+
+// XNACK-LABEL: @rocdl_module
+// XNACK-SAME: [#rocdl.target<triple = "amdgpu9.0a-amd-amdhsa", chip = "gfx90a">]
+// XNACK-SAME: attributes {rocdl.xnack = true}
+
+// BOTH-LABEL: @rocdl_module
+// BOTH-SAME: [#rocdl.target<triple = "amdgpu9.0a-amd-amdhsa", chip = "gfx90a">]
+// BOTH-SAME: attributes {rocdl.sramecc = false, rocdl.xnack = false}
+
+// LEGACY-LABEL: @rocdl_module
+// LEGACY-SAME: [#rocdl.target<chip = "gfx90a">]
+
+// FEATURES-LABEL: @rocdl_module
+// FEATURES-SAME: [#rocdl.target<triple = "amdgpu9.0a-amd-amdhsa", chip = "gfx90a", features = "+dpp">]
+
+// FAMILY-LABEL: @rocdl_module
+// FAMILY-SAME: [#rocdl.target<triple = "amdgpu9.50-amd-amdhsa", chip = "gfx950">]
+
+// GENERIC-LABEL: @rocdl_module
+// GENERIC-SAME: [#rocdl.target<triple = "amdgpu9.4-amd-amdhsa", chip = "gfx9-4-generic">]
+
+// WAVE32-LABEL: @rocdl_module
+// WAVE32-SAME: [#rocdl.target<triple = "amdgpu12.50-amd-amdhsa", chip = "gfx1250", flags = {no_wave64}>]
+
+// WAVE32-RDNA-LABEL: @rocdl_module
+// WAVE32-RDNA-SAME: [#rocdl.target<triple = "amdgpu10.30-amd-amdhsa", chip = "gfx1030", features = "+wavefrontsize32", flags = {no_wave64}>]
+
+// WAVE64-RDNA-LABEL: @rocdl_module
+// WAVE64-RDNA-SAME: [#rocdl.target<triple = "amdgpu10.30-amd-amdhsa", chip = "gfx1030", features = "+wavefrontsize64">]
+gpu.module @rocdl_module {
+}
+
+}
diff --git a/mlir/test/Integration/GPU/ROCM/gpu-lower-to-rocdl-pipeline.mlir b/mlir/test/Integration/GPU/ROCM/gpu-lower-to-rocdl-pipeline.mlir
index ee289b9cd55491..c9d8561d2e82d6 100644
--- a/mlir/test/Integration/GPU/ROCM/gpu-lower-to-rocdl-pipeline.mlir
+++ b/mlir/test/Integration/GPU/ROCM/gpu-lower-to-rocdl-pipeline.mlir
@@ -1,5 +1,5 @@
// RUN: mlir-opt %s \
-// RUN: --gpu-lower-to-rocdl-pipeline="chip=%chip" \
+// RUN: --gpu-lower-to-rocdl-pipeline="arch=%chip" \
// RUN: | mlir-runner \
// RUN: --shared-libs=%mlir_rocm_runtime \
// RUN: --shared-libs=%mlir_runner_utils \
diff --git a/mlir/test/Integration/GPU/ROCM/gpu-to-hsaco.mlir b/mlir/test/Integration/GPU/ROCM/gpu-to-hsaco.mlir
index 5fa27eab3bba46..ea570c2970b71d 100644
--- a/mlir/test/Integration/GPU/ROCM/gpu-to-hsaco.mlir
+++ b/mlir/test/Integration/GPU/ROCM/gpu-to-hsaco.mlir
@@ -1,6 +1,6 @@
// RUN: mlir-opt %s \
// RUN: | mlir-opt -gpu-kernel-outlining \
-// RUN: | mlir-opt -pass-pipeline='builtin.module(gpu.module(strip-debuginfo,convert-gpu-to-rocdl),rocdl-attach-target{chip=%chip})' \
+// RUN: | mlir-opt -pass-pipeline='builtin.module(gpu.module(strip-debuginfo,convert-gpu-to-rocdl{arch=%chip}),rocdl-attach-target{arch=%chip})' \
// RUN: | mlir-opt -gpu-to-llvm -reconcile-unrealized-casts -gpu-module-to-binary \
// RUN: | mlir-runner \
// RUN: --shared-libs=%mlir_rocm_runtime \
diff --git a/mlir/test/Integration/GPU/ROCM/printf.mlir b/mlir/test/Integration/GPU/ROCM/printf.mlir
index 8327ec428589dd..1c67dae8cfe27a 100644
--- a/mlir/test/Integration/GPU/ROCM/printf.mlir
+++ b/mlir/test/Integration/GPU/ROCM/printf.mlir
@@ -1,5 +1,5 @@
// RUN: mlir-opt %s \
-// RUN: | mlir-opt -pass-pipeline='builtin.module(gpu.module(strip-debuginfo,convert-gpu-to-rocdl{index-bitwidth=32 runtime=HIP}),rocdl-attach-target{chip=%chip})' \
+// RUN: | mlir-opt -pass-pipeline='builtin.module(gpu.module(strip-debuginfo,convert-gpu-to-rocdl{arch=%chip index-bitwidth=32 runtime=HIP}),rocdl-attach-target{arch=%chip})' \
// RUN: | mlir-opt -gpu-to-llvm -reconcile-unrealized-casts -gpu-module-to-binary \
// RUN: | mlir-runner \
// RUN: --shared-libs=%mlir_rocm_runtime \
diff --git a/mlir/test/Integration/GPU/ROCM/two-modules.mlir b/mlir/test/Integration/GPU/ROCM/two-modules.mlir
index f3062dbda86c8f..688fd2248b835b 100644
--- a/mlir/test/Integration/GPU/ROCM/two-modules.mlir
+++ b/mlir/test/Integration/GPU/ROCM/two-modules.mlir
@@ -1,6 +1,6 @@
// RUN: mlir-opt %s \
// RUN: | mlir-opt -gpu-kernel-outlining \
-// RUN: | mlir-opt -pass-pipeline='builtin.module(gpu.module(strip-debuginfo,convert-gpu-to-rocdl),rocdl-attach-target{chip=%chip})' \
+// RUN: | mlir-opt -pass-pipeline='builtin.module(gpu.module(strip-debuginfo,convert-gpu-to-rocdl{arch=%chip}),rocdl-attach-target{arch=%chip})' \
// RUN: | mlir-opt -gpu-to-llvm -reconcile-unrealized-casts -gpu-module-to-binary \
// RUN: | mlir-runner \
// RUN: --shared-libs=%mlir_rocm_runtime \
diff --git a/mlir/test/Integration/GPU/ROCM/vecadd.mlir b/mlir/test/Integration/GPU/ROCM/vecadd.mlir
index c3f8b982a131a1..88a969fef82018 100644
--- a/mlir/test/Integration/GPU/ROCM/vecadd.mlir
+++ b/mlir/test/Integration/GPU/ROCM/vecadd.mlir
@@ -1,7 +1,7 @@
// RUN: mlir-opt %s \
// RUN: | mlir-opt -convert-scf-to-cf \
// RUN: | mlir-opt -gpu-kernel-outlining \
-// RUN: | mlir-opt -pass-pipeline='builtin.module(gpu.module(strip-debuginfo,convert-gpu-to-rocdl{use-bare-ptr-memref-call-conv=true}),rocdl-attach-target{chip=%chip})' \
+// RUN: | mlir-opt -pass-pipeline='builtin.module(gpu.module(strip-debuginfo,convert-gpu-to-rocdl{arch=%chip use-bare-ptr-memref-call-conv=true}),rocdl-attach-target{arch=%chip})' \
// RUN: | mlir-opt -gpu-to-llvm=use-bare-pointers-for-kernels=true -reconcile-unrealized-casts -gpu-module-to-binary \
// RUN: | mlir-runner \
// RUN: --shared-libs=%mlir_rocm_runtime \
diff --git a/mlir/test/Integration/GPU/ROCM/vector-transferops.mlir b/mlir/test/Integration/GPU/ROCM/vector-transferops.mlir
index a633edb8377af9..6c13ded50307c1 100644
--- a/mlir/test/Integration/GPU/ROCM/vector-transferops.mlir
+++ b/mlir/test/Integration/GPU/ROCM/vector-transferops.mlir
@@ -1,7 +1,7 @@
// RUN: mlir-opt %s \
// RUN: | mlir-opt -convert-scf-to-cf \
// RUN: | mlir-opt -gpu-kernel-outlining \
-// RUN: | mlir-opt -pass-pipeline='builtin.module(gpu.module(strip-debuginfo,convert-gpu-to-rocdl{chipset=%chip index-bitwidth=32}),rocdl-attach-target{chip=%chip})' \
+// RUN: | mlir-opt -pass-pipeline='builtin.module(gpu.module(strip-debuginfo,convert-gpu-to-rocdl{arch=%chip index-bitwidth=32}),rocdl-attach-target{arch=%chip})' \
// RUN: | mlir-opt -gpu-to-llvm -reconcile-unrealized-casts -gpu-module-to-binary \
// RUN: | mlir-runner \
// RUN: --shared-libs=%mlir_rocm_runtime \
diff --git a/mlir/test/lib/Dialect/GPU/TestGpuRewrite.cpp b/mlir/test/lib/Dialect/GPU/TestGpuRewrite.cpp
index 72c3952f0f496b..4b620496d6b410 100644
--- a/mlir/test/lib/Dialect/GPU/TestGpuRewrite.cpp
+++ b/mlir/test/lib/Dialect/GPU/TestGpuRewrite.cpp
@@ -11,7 +11,6 @@
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
-#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncDialect.h"
#include "mlir/Dialect/GPU/Transforms/Passes.h"
@@ -91,12 +90,16 @@ struct TestGpuSubgroupReduceLoweringPass
/*maxShuffleBitwidth=*/32,
PatternBenefit(3));
if (expandToShuffles) {
- auto maybeChipset = amdgpu::Chipset::parse(target);
- if (succeeded(maybeChipset)) {
+ if (!target.empty()) {
+ FailureOr<ROCDL::TargetInfo> targetInfo =
+ ROCDL::TargetInfo::get(target, /*waveSize=*/0,
+ [&] { return getOperation()->emitError(); });
+ if (failed(targetInfo))
+ return signalPassFailure();
populateGpuLowerSubgroupReduceToDPPPatterns(
- patterns, /*subgroupSize=*/64, *maybeChipset, PatternBenefit(2));
+ patterns, /*subgroupSize=*/64, *targetInfo, PatternBenefit(2));
populateGpuLowerClusteredSubgroupReduceToDPPPatterns(
- patterns, /*subgroupSize=*/64, *maybeChipset, PatternBenefit(2));
+ patterns, /*subgroupSize=*/64, *targetInfo, PatternBenefit(2));
}
populateGpuLowerSubgroupReduceToShufflePatterns(
patterns, /*subgroupSize=*/32, /*shuffleBitwidth=*/32);
diff --git a/mlir/unittests/Dialect/AMDGPU/AMDGPUUtilsTest.cpp b/mlir/unittests/Dialect/AMDGPU/AMDGPUUtilsTest.cpp
index 570d56f3c6ff13..9b22f450031d5b 100644
--- a/mlir/unittests/Dialect/AMDGPU/AMDGPUUtilsTest.cpp
+++ b/mlir/unittests/Dialect/AMDGPU/AMDGPUUtilsTest.cpp
@@ -7,8 +7,13 @@
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
+#include "llvm/Support/Compiler.h"
#include "gtest/gtest.h"
+// Chipset is deprecated in favour of ROCDL::TargetInfo, but stays covered for
+// as long as it ships.
+LLVM_SUPPRESS_DEPRECATED_DECLARATIONS_PUSH
+
namespace mlir::amdgpu {
namespace {
@@ -60,3 +65,5 @@ TEST(ChipsetTest, Comparison) {
} // namespace
} // namespace mlir::amdgpu
+
+LLVM_SUPPRESS_DEPRECATED_DECLARATIONS_POP
>From b5c5153a008d9e64cb08bb3431e6360674b9e29a Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston at google.com>
Date: Tue, 6 Oct 2026 15:10:37 -0700
Subject: [PATCH 42/46] [msan][test] Add MSan case to
clang/test/CodeGen/fake-use-sanitizer.cpp (#229564)
Regression test for https://github.com/llvm/llvm-project/issues/225425,
showing that MSan incorrectly strictly handles fake_use.
---
clang/test/CodeGen/fake-use-sanitizer.cpp | 46 +++++++++++++++++++++++
1 file changed, 46 insertions(+)
diff --git a/clang/test/CodeGen/fake-use-sanitizer.cpp b/clang/test/CodeGen/fake-use-sanitizer.cpp
index e808c645b9c60f..68a702b1326635 100644
--- a/clang/test/CodeGen/fake-use-sanitizer.cpp
+++ b/clang/test/CodeGen/fake-use-sanitizer.cpp
@@ -1,4 +1,5 @@
// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -fextend-variable-liveness -fsanitize=null -fsanitize-trap=null -o - | FileCheck --check-prefixes=CHECK,NULL --implicit-check-not=ubsantrap %s
+// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -fextend-variable-liveness -fsanitize=memory -o - | FileCheck %s --check-prefixes=MEMORY
// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -fextend-variable-liveness -o - | FileCheck %s
// With -fextend-variable-liveness, the compiler previously generated a fake.use of any
@@ -59,3 +60,48 @@ void foo()
// CHECK-NEXT: call void (...) @llvm.fake.use(ptr [[VA_FAKE_USE]])
// NULL: declare void @llvm.ubsantrap
+
+
+// MSan regression test from https://github.com/llvm/llvm-project/issues/225425
+// This test (not the tests above) was generated with update_cc_test_checks.py.
+//
+// A fake_use will be inserted for x. Currently, MSan erroneously checks it.
+//
+// MEMORY-LABEL: define dso_local noundef i32 @main(
+// MEMORY-SAME: ) #[[ATTR0:[0-9]+]] {
+// MEMORY-NEXT: [[ENTRY:.*:]]
+// MEMORY-NEXT: call void @llvm.donothing()
+// MEMORY-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
+// MEMORY-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[RETVAL]] to i64
+// MEMORY-NEXT: [[TMP1:%.*]] = xor i64 [[TMP0]], 87960930222080
+// MEMORY-NEXT: [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+// MEMORY-NEXT: call void @llvm.memset.p0.i64(ptr align 4 [[TMP2]], i8 -1, i64 4, i1 false)
+// MEMORY-NEXT: [[X:%.*]] = alloca i8, align 1
+// MEMORY-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[RETVAL]] to i64
+// MEMORY-NEXT: [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+// MEMORY-NEXT: [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+// MEMORY-NEXT: store i32 0, ptr [[TMP5]], align 4
+// MEMORY-NEXT: store i32 0, ptr [[RETVAL]], align 4
+// MEMORY-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]]) #[[ATTR6:[0-9]+]]
+// MEMORY-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[X]] to i64
+// MEMORY-NEXT: [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+// MEMORY-NEXT: [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+// MEMORY-NEXT: call void @llvm.memset.p0.i64(ptr align 1 [[TMP8]], i8 -1, i64 1, i1 false)
+// MEMORY-NEXT: [[FAKE_USE:%.*]] = load i8, ptr [[X]], align 1
+// MEMORY-NEXT: [[TMP9:%.*]] = ptrtoint ptr [[X]] to i64
+// MEMORY-NEXT: [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+// MEMORY-NEXT: [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+// MEMORY-NEXT: [[_MSLD:%.*]] = load i8, ptr [[TMP11]], align 1
+// MEMORY-NEXT: [[_MSCMP:%.*]] = icmp ne i8 [[_MSLD]], 0
+// MEMORY-NEXT: br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF2:![0-9]+]]
+// MEMORY: [[BB12]]:
+// MEMORY-NEXT: call void @__msan_warning_noreturn() #[[ATTR7:[0-9]+]]
+// MEMORY-NEXT: unreachable
+// MEMORY: [[BB13]]:
+// MEMORY-NEXT: notail call void (...) @llvm.fake.use(i8 [[FAKE_USE]]) #[[ATTR6]]
+// MEMORY-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]]) #[[ATTR6]]
+// MEMORY-NEXT: ret i32 0
+int main() {
+ unsigned char x;
+ return 0;
+}
>From 3a01c7bd535af2bd9817fc1862dd892c7f731ecc Mon Sep 17 00:00:00 2001
From: Bruno Cardoso Lopes <bruno.cardoso at gmail.com>
Date: Tue, 6 Oct 2026 15:13:37 -0700
Subject: [PATCH 43/46] [CIR] Fix use-after-free in BrOp::canonicalize
(#229191)
BrOp::canonicalize took the branch's destination operands as an
OperandRange, erased the branch, and then passed the range to
mergeBlocks. The range points into the branch's operand storage, which
eraseOp frees, so mergeBlocks read freed memory whenever the merged
block had arguments. Valgrind reports it on the new test:
```
Invalid read of size 8
at mlir::ValueRange::dereference_iterator
by mlir::RewriterBase::inlineBlockBefore
by cir::BrOp::canonicalize
Address ... is 136 bytes inside a block of size 144 free'd
by mlir::RewriterBase::eraseOp
by cir::BrOp::canonicalize
```
It goes unnoticed with glibc malloc, which usually leaves the freed
operands intact. Copy the operands before erasing the branch.
---
clang/lib/CIR/Dialect/IR/CIRDialect.cpp | 12 +++++++-
clang/test/CIR/Transforms/canonicalize.cir | 34 ++++++++++++++++++++++
2 files changed, 45 insertions(+), 1 deletion(-)
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index 51b5bd1c58f4bd..2a5cb9a908a1dc 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -2067,7 +2067,17 @@ LogicalResult cir::BrOp::canonicalize(BrOp op, PatternRewriter &rewriter) {
if (isa<cir::LabelOp, cir::IndirectBrOp>(dst->front()))
return failure();
- auto operands = op.getDestOperands();
+ // An operand that is an argument of the destination itself (possible in an
+ // unreachable cycle) would be replaced by itself, leaving its other uses
+ // dangling once the destination is erased.
+ if (llvm::any_of(op.getDestOperands(), [&](Value operand) {
+ auto arg = dyn_cast<BlockArgument>(operand);
+ return arg && arg.getOwner() == dst;
+ }))
+ return failure();
+
+ // Copy the operands out: erasing the branch frees its operand storage.
+ SmallVector<Value> operands(op.getDestOperands());
rewriter.eraseOp(op);
rewriter.mergeBlocks(dst, src, operands);
return success();
diff --git a/clang/test/CIR/Transforms/canonicalize.cir b/clang/test/CIR/Transforms/canonicalize.cir
index 29b1d880a37489..a786b252e91ad6 100644
--- a/clang/test/CIR/Transforms/canonicalize.cir
+++ b/clang/test/CIR/Transforms/canonicalize.cir
@@ -28,6 +28,40 @@ module {
// CHECK-NEXT: cir.return
// CHECK-NEXT: }
+ // The branch operands replace the arguments of the merged blocks.
+ cir.func @redundant_br_block_args(%arg0: !s32i, %arg1: !s32i) -> !s32i {
+ cir.br ^bb1(%arg0, %arg1 : !s32i, !s32i)
+ ^bb1(%0: !s32i, %1: !s32i): // pred: ^bb0
+ %2 = cir.add %0, %1 : !s32i
+ cir.br ^bb2(%2 : !s32i)
+ ^bb2(%3: !s32i): // pred: ^bb1
+ cir.return %3 : !s32i
+ }
+ // CHECK: cir.func{{.*}} @redundant_br_block_args(%[[ARG0:.*]]: !s32i, %[[ARG1:.*]]: !s32i) -> !s32i {
+ // CHECK-NEXT: %[[SUM:.*]] = cir.add %[[ARG0]], %[[ARG1]] : !s32i
+ // CHECK-NEXT: cir.return %[[SUM]] : !s32i
+ // CHECK-NEXT: }
+
+ // In this unreachable cycle, ^header is the only predecessor of ^body and
+ // ^body the only predecessor of ^header, and ^body passes ^header's own
+ // argument back to it. ^header must not be merged into ^body: %keep would be
+ // replaced by itself, and its use in the call would outlive ^header.
+ cir.func private @use(!s32i)
+ cir.func @no_merge_self_arg_loop(%arg0: !s32i) -> !s32i {
+ cir.return %arg0 : !s32i
+ ^header(%keep: !s32i):
+ cir.call @use(%keep) : (!s32i) -> ()
+ cir.br ^body
+ ^body:
+ cir.br ^header(%keep : !s32i)
+ }
+ // CHECK: cir.func{{.*}} @no_merge_self_arg_loop(%[[ARG0:.*]]: !s32i) -> !s32i {
+ // CHECK-NEXT: cir.return %[[ARG0]] : !s32i
+ // CHECK-NEXT: ^[[HEADER:bb[0-9]+]](%[[KEEP:.*]]: !s32i):
+ // CHECK-NEXT: cir.call @use(%[[KEEP]]) : (!s32i) -> ()
+ // CHECK-NEXT: cir.br ^[[HEADER]](%[[KEEP]] : !s32i)
+ // CHECK-NEXT: }
+
cir.func @scope_yield_value_fold() -> !u32i {
%0 = cir.const #cir.int<7> : !u32i
%1 = cir.scope {
>From 1d653763f859d2e87ad9e7c3656b61584e4ae8a2 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 6 Oct 2026 23:38:36 +0100
Subject: [PATCH 44/46] [LV] Use frozen start values from the main plan
directly (NFC). (#229571)
Only freeze possibly-poison start values of FindIV reductions in the
main plan. When preparing the epilogue plan, set the start operand of
its FindIV reduction results directly to the value frozen for the main
plan, instead of creating Freeze recipes in the epilogue plan and
replacing them later via a map. This leaves only VPExpandSCEVRecipes to
replace in the epilogue plan's entry.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 85 ++++++++-----------
1 file changed, 34 insertions(+), 51 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 62a4f7a74ee189..3cdc144bace673 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7283,33 +7283,27 @@ struct MainPlanResumeMarkers {
/// Prepare \p MainPlan for vectorizing the main vector loop during epilogue
/// vectorization.
-static MainPlanResumeMarkers preparePlanForMainVectorLoop(VPlan &MainPlan,
- VPlan &EpiPlan) {
+static MainPlanResumeMarkers preparePlanForMainVectorLoop(VPlan &MainPlan) {
using namespace VPlanPatternMatch;
// When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
// introduce multiple uses of undef/poison. If the reduction start value may
// be undef or poison it needs to be frozen and the frozen start has to be
// used when computing the reduction result. We also need to use the frozen
// value in the resume phi generated by the main vector loop, as this is also
- // used to compute the reduction result after the epilogue vector loop.
- auto AddFreezeForFindLastIVReductions = [](VPlan &Plan,
- bool UpdateResumePhis) {
- VPBuilder Builder(Plan.getEntry());
- for (VPInstruction &VPI :
- make_isa_range<VPInstruction>(*Plan.getMiddleBlock())) {
- VPValue *OrigStart;
- if (!matchFindIVResult(&VPI, m_VPValue(), m_VPValue(OrigStart)))
- continue;
- if (isGuaranteedNotToBeUndefOrPoison(OrigStart->getLiveInIRValue()))
- continue;
- VPInstruction *Freeze = Builder.createFreeze(OrigStart, {}, "fr");
- VPI.setOperand(2, Freeze);
- if (UpdateResumePhis)
- OrigStart->replaceUsesWithIf(Freeze, IsaPred<VPPhi>);
- }
- };
- AddFreezeForFindLastIVReductions(MainPlan, true);
- AddFreezeForFindLastIVReductions(EpiPlan, false);
+ // used to compute the reduction result after the epilogue vector loop. The
+ // epilogue plan re-uses the frozen values.
+ VPBuilder Builder(MainPlan.getEntry());
+ for (VPInstruction &VPI :
+ make_isa_range<VPInstruction>(*MainPlan.getMiddleBlock())) {
+ VPValue *OrigStart;
+ if (!matchFindIVResult(&VPI, m_VPValue(), m_VPValue(OrigStart)))
+ continue;
+ if (isGuaranteedNotToBeUndefOrPoison(OrigStart->getLiveInIRValue()))
+ continue;
+ VPInstruction *Freeze = Builder.createFreeze(OrigStart, {}, "fr");
+ VPI.setOperand(2, Freeze);
+ OrigStart->replaceUsesWithIf(Freeze, IsaPred<VPPhi>);
+ }
VPValue *VectorTC = nullptr;
auto *Term =
@@ -7417,8 +7411,6 @@ static void preparePlanForEpilogueVectorLoop(
Increment->replaceAllUsesWith(OffsetIVInc);
OffsetIVInc->setOperand(0, Increment);
- SmallDenseMap<VPValue *, VPValue *, 8> ToFrozen;
-
// Resume values must be created in the vector preheader.
VPBasicBlock *VectorPH = Plan.getVectorPreheader();
VPBuilder PHBuilder(VectorPH, VectorPH->getFirstNonPhi());
@@ -7470,10 +7462,14 @@ static void preparePlanForEpilogueVectorLoop(
} else {
assert(isa<VPIRValue>(SentinelVPV) &&
"sentinel must be a live-in to be used in the preheader");
- VPValue *OrigStart;
- if (VPlanPatternMatch::match(
- BypassOp, VPlanPatternMatch::m_Freeze(m_VPValue(OrigStart))))
- ToFrozen[Plan.getOrAddLiveIn(cast<VPIRValue>(OrigStart))] = StartV;
+ // Use the start value frozen for the main plan, if any, when
+ // computing the reduction result.
+ if (match(BypassOp, m_Freeze(m_VPValue())))
+ for (VPUser *U : RdxResult->users()) {
+ auto *VPI = dyn_cast<VPInstruction>(U);
+ if (VPI && matchFindIVResult(VPI, m_VPValue(), m_VPValue()))
+ VPI->setOperand(2, StartV);
+ }
// Adjust resume: select(icmp eq ResumeVPV, StartV), Sentinel,
// ResumeVPV
@@ -7530,34 +7526,22 @@ static void preparePlanForEpilogueVectorLoop(
cast<VPHeaderPHIRecipe>(&R)->setStartValue(ResumeVPV);
}
- // For some VPValues in the epilogue plan we must re-use the generated IR
- // values from the main plan. Replace them with live-in VPValues.
+ // Re-use the trip count and steps expanded for the main loop, as skeleton
+ // creation needs it as a value that dominates both the scalar and vector
+ // epilogue loops.
// TODO: This is a workaround needed for epilogue vectorization and it
// should be removed once induction resume value creation is done
// directly in VPlan.
- for (auto &R : make_early_inc_range(*Plan.getEntry())) {
- // Re-use frozen values from the main plan for Freeze VPInstructions in the
- // epilogue plan. This ensures all users use the same frozen value.
- auto *VPI = dyn_cast<VPInstruction>(&R);
- if (VPI && VPI->getOpcode() == Instruction::Freeze) {
- VPI->replaceAllUsesWith(ToFrozen.lookup(VPI->getOperand(0)));
- continue;
- }
-
- // Re-use the trip count and steps expanded for the main loop, as
- // skeleton creation needs it as a value that dominates both the scalar
- // and vector epilogue loops
- auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
- if (!ExpandR)
- continue;
- assert(ExpandedSCEVs.contains(ExpandR->getSCEV()) &&
+ for (VPExpandSCEVRecipe &ExpandR : make_early_inc_range(
+ make_isa_range<VPExpandSCEVRecipe>(*Plan.getEntry()))) {
+ assert(ExpandedSCEVs.contains(ExpandR.getSCEV()) &&
"Epilogue plan needs a SCEV not expanded for the main loop");
VPValue *ExpandedVal =
- Plan.getOrAddLiveIn(ExpandedSCEVs.lookup(ExpandR->getSCEV()));
- ExpandR->replaceAllUsesWith(ExpandedVal);
- if (Plan.getTripCount() == ExpandR)
+ Plan.getOrAddLiveIn(ExpandedSCEVs.lookup(ExpandR.getSCEV()));
+ ExpandR.replaceAllUsesWith(ExpandedVal);
+ if (Plan.getTripCount() == &ExpandR)
Plan.resetTripCount(ExpandedVal);
- ExpandR->eraseFromParent();
+ ExpandR.eraseFromParent();
}
auto VScale = Config.getVScaleForTuning();
@@ -8084,8 +8068,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// factor) again shortly afterwards.
BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
BestEpiPlan.getVectorPreheader()->setName("vec.epilog.ph");
- MainPlanResumeMarkers Markers =
- preparePlanForMainVectorLoop(BestMainPlan, BestEpiPlan);
+ MainPlanResumeMarkers Markers = preparePlanForMainVectorLoop(BestMainPlan);
// Add minimum iteration check for the epilogue plan, followed by runtime
// checks for the main plan.
>From d62290f3980cb70cddc8124db562c4f34cbef0e1 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper at sifive.com>
Date: Tue, 6 Oct 2026 15:43:58 -0700
Subject: [PATCH 45/46] [RISCV] Fold (sub 0, (srl (and X, (1 << ShAmt)),
ShAmt)) -> (sra (shl X, ShAmt2), bits-1) (#229509)
Improves codegen of is.fpclass+select. The AND to test bits of
is.fpclass may get turned into and+srl while the select emits a
neg. Isel will turn the and+srl into shl+srl but it's too late to
fold the neg.
Assisted-by: Claude
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 23 ++++
llvm/test/CodeGen/RISCV/fclass-select.ll | 114 ++++++++++++++++++++
2 files changed, 137 insertions(+)
create mode 100644 llvm/test/CodeGen/RISCV/fclass-select.ll
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 4d7874fed7c2a8..378379e0fc7eef 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -19495,6 +19495,29 @@ static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
}
}
+ // fold (sub 0, (srl (and X, (1 << ShAmt)), ShAmt)) ->
+ // (sra (shl X, ShAmt2), bits - 1)
+ // where ShAmt2 = (bits - 1) - ShAmt. This extracts bit ShAmt of X and
+ // sign-extends it across the whole register.
+ {
+ using namespace SDPatternMatch;
+ SDValue X;
+ uint64_t Mask, ShAmt;
+ if (isNullConstant(N0) &&
+ sd_match(N1,
+ m_OneUse(m_Srl(m_OneUse(m_And(m_Value(X), m_ConstInt(Mask))),
+ m_ConstInt(ShAmt)))) &&
+ isPowerOf2_64(Mask) && Log2_64(Mask) == ShAmt) {
+ SDLoc DL(N);
+ unsigned Bits = VT.getSizeInBits();
+ unsigned ShAmt2 = Bits - 1 - ShAmt;
+ SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, X,
+ DAG.getShiftAmountConstant(ShAmt2, VT, DL));
+ return DAG.getNode(ISD::SRA, DL, VT, Shl,
+ DAG.getShiftAmountConstant(Bits - 1, VT, DL));
+ }
+ }
+
if (SDValue V = combinePExtWideningSubAcc(N, DAG, Subtarget))
return V;
if (SDValue V = combinePExtWideningAddSub(N, DAG, Subtarget))
diff --git a/llvm/test/CodeGen/RISCV/fclass-select.ll b/llvm/test/CodeGen/RISCV/fclass-select.ll
new file mode 100644
index 00000000000000..34e54f1a8c79a6
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/fclass-select.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -mattr=+d < %s | FileCheck %s --check-prefix=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+d < %s | FileCheck %s --check-prefix=RV64
+
+; Fold (sub 0, (srl (and X, (1 << ShAmt)), ShAmt)) ->
+; (sra (shl X, bits-1-ShAmt), bits-1)
+; This turns an and+srl+neg sequence into a sign-extension of a single bit
+; via shl+sra. llvm.is.fpclass with a single-bit mask is a convenient way to
+; generate the and+srl+sub pattern.
+
+define i32 @is_snan_sext_f64(double %x) nounwind {
+; RV32-LABEL: is_snan_sext_f64:
+; RV32: # %bb.0:
+; RV32-NEXT: fclass.d a0, fa0
+; RV32-NEXT: slli a0, a0, 23
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: ret
+;
+; RV64-LABEL: is_snan_sext_f64:
+; RV64: # %bb.0:
+; RV64-NEXT: fclass.d a0, fa0
+; RV64-NEXT: slli a0, a0, 55
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: ret
+ %c = call i1 @llvm.is.fpclass.f64(double %x, i32 1) ; fcSNan
+ %s = select i1 %c, i32 -1, i32 0
+ ret i32 %s
+}
+
+define i32 @is_snan_sext_f32(float %x) nounwind {
+; RV32-LABEL: is_snan_sext_f32:
+; RV32: # %bb.0:
+; RV32-NEXT: fclass.s a0, fa0
+; RV32-NEXT: slli a0, a0, 23
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: ret
+;
+; RV64-LABEL: is_snan_sext_f32:
+; RV64: # %bb.0:
+; RV64-NEXT: fclass.s a0, fa0
+; RV64-NEXT: slli a0, a0, 55
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: ret
+ %c = call i1 @llvm.is.fpclass.f32(float %x, i32 1) ; fcSNan
+ %s = select i1 %c, i32 -1, i32 0
+ ret i32 %s
+}
+
+; The and gets combined away by DAGCombiner before performSUBCombine runs
+; (it can prove the high bits are already zero), so this case doesn't match
+; the and+srl pattern and keeps the srli+neg form instead.
+define i32 @is_qnan_sext_f64(double %x) nounwind {
+; RV32-LABEL: is_qnan_sext_f64:
+; RV32: # %bb.0:
+; RV32-NEXT: fclass.d a0, fa0
+; RV32-NEXT: srli a0, a0, 9
+; RV32-NEXT: neg a0, a0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: is_qnan_sext_f64:
+; RV64: # %bb.0:
+; RV64-NEXT: fclass.d a0, fa0
+; RV64-NEXT: srli a0, a0, 9
+; RV64-NEXT: neg a0, a0
+; RV64-NEXT: ret
+ %c = call i1 @llvm.is.fpclass.f64(double %x, i32 2) ; fcQNan
+ %s = select i1 %c, i32 -1, i32 0
+ ret i32 %s
+}
+
+define i32 @is_posinf_sext_f64(double %x) nounwind {
+; RV32-LABEL: is_posinf_sext_f64:
+; RV32: # %bb.0:
+; RV32-NEXT: fclass.d a0, fa0
+; RV32-NEXT: slli a0, a0, 24
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: ret
+;
+; RV64-LABEL: is_posinf_sext_f64:
+; RV64: # %bb.0:
+; RV64-NEXT: fclass.d a0, fa0
+; RV64-NEXT: slli a0, a0, 56
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: ret
+ %c = call i1 @llvm.is.fpclass.f64(double %x, i32 512) ; fcPosInf
+ %s = select i1 %c, i32 -1, i32 0
+ ret i32 %s
+}
+
+; A multi-bit mask isn't a single-bit extraction, so this should not fold to
+; the shl+sra pattern.
+define i32 @is_nan_sext_f64_no_fold(double %x) nounwind {
+; RV32-LABEL: is_nan_sext_f64_no_fold:
+; RV32: # %bb.0:
+; RV32-NEXT: fclass.d a0, fa0
+; RV32-NEXT: andi a0, a0, 768
+; RV32-NEXT: seqz a0, a0
+; RV32-NEXT: addi a0, a0, -1
+; RV32-NEXT: ret
+;
+; RV64-LABEL: is_nan_sext_f64_no_fold:
+; RV64: # %bb.0:
+; RV64-NEXT: fclass.d a0, fa0
+; RV64-NEXT: andi a0, a0, 768
+; RV64-NEXT: seqz a0, a0
+; RV64-NEXT: addi a0, a0, -1
+; RV64-NEXT: ret
+ %c = call i1 @llvm.is.fpclass.f64(double %x, i32 3) ; fcNan (two bits)
+ %s = select i1 %c, i32 -1, i32 0
+ ret i32 %s
+}
+
+declare i1 @llvm.is.fpclass.f64(double, i32)
+declare i1 @llvm.is.fpclass.f32(float, i32)
>From dbee642ac228444bf8c76e3ca0e6281973ac92b1 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 6 Oct 2026 23:47:43 +0000
Subject: [PATCH 46/46] ir-translator pass name change
---
llvm/test/CodeGen/X86/GlobalISel/irtranslator-mem-cache-hint.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/X86/GlobalISel/irtranslator-mem-cache-hint.ll b/llvm/test/CodeGen/X86/GlobalISel/irtranslator-mem-cache-hint.ll
index b0e2a3289dee00..da9c9282e0a714 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/irtranslator-mem-cache-hint.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/irtranslator-mem-cache-hint.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=x86_64-linux-gnu -O0 -global-isel \
-; RUN: -stop-after=irtranslator %s -o - | FileCheck %s
+; RUN: -stop-after=ir-translator %s -o - | FileCheck %s
; CHECK-DAG: !1 = !{!"test.cache", !"load"}
; CHECK-DAG: !3 = !{!"test.cache", !"store"}
More information about the flang-commits
mailing list