[llvm-branch-commits] [llvm] AMDGPU: Promote 16-bit atomic load/store to extending i32 (PR #230943)
Matt Arsenault via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Oct 11 03:35:58 PDT 2026
https://github.com/arsenm created https://github.com/llvm/llvm-project/pull/230943
16-bit atomic loads are really extending loads into a 32-bit register,
and 16-bit atomic stores are truncating stores of a 32-bit value. Without
real true16, promote i16, f16 and bf16 atomic load and store to i32 as
any-extending loads and truncating stores, matching how regular loads
and stores are handled. With real true16, select all 16-bit types
directly using the type-generic atomic patterns, as is done for the
wider types.
Teach LegalizeDAG to promote ATOMIC_LOAD and ATOMIC_STORE to a wider
integer type, preserving the extension type. The i16-only atomic
patterns are now only needed for real true16, so remove the rest.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
>From b6c7cf8f374960f650a068f125189fe3d6ec0203 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Thu, 8 Oct 2026 20:24:06 +0200
Subject: [PATCH] AMDGPU: Promote 16-bit atomic load/store to extending i32
16-bit atomic loads are really extending loads into a 32-bit register,
and 16-bit atomic stores are truncating stores of a 32-bit value. Without
real true16, promote i16, f16 and bf16 atomic load and store to i32 as
any-extending loads and truncating stores, matching how regular loads
and stores are handled. With real true16, select all 16-bit types
directly using the type-generic atomic patterns, as is done for the
wider types.
Teach LegalizeDAG to promote ATOMIC_LOAD and ATOMIC_STORE to a wider
integer type, preserving the extension type. The i16-only atomic
patterns are now only needed for real true16, so remove the rest.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 28 +++++++++
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 9 ++-
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 13 ----
llvm/lib/Target/AMDGPU/BUFInstructions.td | 11 +---
llvm/lib/Target/AMDGPU/DSInstructions.td | 61 +++++++++++--------
llvm/lib/Target/AMDGPU/FLATInstructions.td | 35 +++++------
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 8 +++
llvm/test/CodeGen/AMDGPU/gds-load-store.ll | 6 --
8 files changed, 91 insertions(+), 80 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index ffc9173e767fa..dbb7811b0a044 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -6220,6 +6220,18 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
case ISD::ATOMIC_STORE: {
AtomicSDNode *AM = cast<AtomicSDNode>(Node);
SDLoc SL(Node);
+ if (AM->getOpcode() == ISD::ATOMIC_STORE && NVT.bitsGT(OVT)) {
+ assert(NVT.isInteger() && "unexpected promotion type");
+ MVT IntVT = MVT::getIntegerVT(OVT.getSizeInBits());
+ EVT MemVT = AM->getMemoryVT().changeTypeToInteger();
+ SDValue IntVal = DAG.getNode(ISD::BITCAST, SL, IntVT, AM->getVal());
+ SDValue ExtVal = DAG.getNode(ISD::ANY_EXTEND, SL, NVT, IntVal);
+ Results.push_back(DAG.getAtomic(ISD::ATOMIC_STORE, SL, MemVT,
+ AM->getChain(), ExtVal, AM->getBasePtr(),
+ AM->getMemOperand()));
+ break;
+ }
+
SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
"unexpected promotion type");
@@ -6247,6 +6259,22 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
case ISD::ATOMIC_LOAD: {
AtomicSDNode *AM = cast<AtomicSDNode>(Node);
SDLoc SL(Node);
+ if (NVT.bitsGT(OVT)) {
+ assert(NVT.isInteger() && "unexpected promotion type");
+ MVT IntVT = MVT::getIntegerVT(OVT.getSizeInBits());
+ EVT MemVT = AM->getMemoryVT().changeTypeToInteger();
+ ISD::LoadExtType ExtType = AM->getExtensionType();
+ if (ExtType == ISD::NON_EXTLOAD)
+ ExtType = ISD::EXTLOAD;
+ SDValue NewAtomic =
+ DAG.getAtomicLoad(ExtType, SL, MemVT, NVT, AM->getChain(),
+ AM->getBasePtr(), AM->getMemOperand());
+ SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, IntVT, NewAtomic);
+ Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, Trunc));
+ Results.push_back(NewAtomic.getValue(1));
+ break;
+ }
+
assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
"unexpected promotion type");
assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index a2e6a4bf485b5..dc2abb60567ff 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -355,13 +355,12 @@ bool AMDGPUDAGToDAGISel::widenRegionLoad16(SDNode *N) const {
if (ExtType == ISD::NON_EXTLOAD)
ExtType = ISD::EXTLOAD;
+ EVT MemVT = Mem->getMemoryVT().changeTypeToInteger();
SDValue NewLoad =
Ld ? CurDAG->getExtLoad(ExtType, SL, MVT::i32, Mem->getChain(),
- Mem->getBasePtr(), Mem->getMemoryVT(),
- Mem->getMemOperand())
- : CurDAG->getAtomicLoad(ExtType, SL, Mem->getMemoryVT(), MVT::i32,
- Mem->getChain(), Mem->getBasePtr(),
- Mem->getMemOperand());
+ Mem->getBasePtr(), MemVT, Mem->getMemOperand())
+ : CurDAG->getAtomicLoad(ExtType, SL, MemVT, MVT::i32, Mem->getChain(),
+ Mem->getBasePtr(), Mem->getMemOperand());
SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, SL, MVT::i16, NewLoad);
SDValue Ops[] = {CurDAG->getBitcast(VT, Trunc), NewLoad.getValue(1)};
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 3d497278ec6f5..c9ef618e1e7b3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -157,19 +157,6 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::i128, Promote);
AddPromotedToType(ISD::LOAD, MVT::i128, MVT::v4i32);
- // TODO: Would be better to consume as directly legal
- setOperationAction(ISD::ATOMIC_LOAD, MVT::f16, Promote);
- AddPromotedToType(ISD::ATOMIC_LOAD, MVT::f16, MVT::i16);
-
- setOperationAction(ISD::ATOMIC_LOAD, MVT::bf16, Promote);
- AddPromotedToType(ISD::ATOMIC_LOAD, MVT::bf16, MVT::i16);
-
- setOperationAction(ISD::ATOMIC_STORE, MVT::f16, Promote);
- AddPromotedToType(ISD::ATOMIC_STORE, MVT::f16, MVT::i16);
-
- setOperationAction(ISD::ATOMIC_STORE, MVT::bf16, Promote);
- AddPromotedToType(ISD::ATOMIC_STORE, MVT::bf16, MVT::i16);
-
// There are no 64-bit extloads. These should be done as a 32-bit extload and
// an extension to 64-bit.
for (MVT VT : MVT::integer_valuetypes())
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index e7f0c934eb9cc..3a428e1923f77 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -1306,16 +1306,11 @@ defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores <
} // End HasD16LoadStore
-let True16Predicate = NotUseRealTrue16Insts in {
-defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_aext_8_global>;
-defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_zext_8_global>;
-defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_nonext_16_global>;
-}
-
let True16Predicate = UseRealTrue16Insts in {
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE_D16_t16", i16, atomic_load_aext_8_global>;
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE_D16_t16", i16, atomic_load_zext_8_global>;
-defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SHORT_D16_t16", i16, atomic_load_nonext_16_global>;
+foreach vt = Reg16Types.types in
+defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SHORT_D16_t16", vt, atomic_load_global>;
}
let SubtargetPredicate = isNotGFX940Plus in
@@ -2239,9 +2234,7 @@ multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo In
}
let SubtargetPredicate = isGFX6GFX7 in {
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>;
-defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>;
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>;
-defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>;
} // End Predicates = isGFX6GFX7
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index 3ff2ef169ee8f..7f123ad962e4d 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -965,6 +965,15 @@ multiclass DSReadPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
}
}
+multiclass DSReadPat_True16<DS_Pseudo inst, ValueType vt, string frag> {
+ let OtherPredicates = [NotLDSRequiresM0Init, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts in {
+ def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>;
+ }
+ let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = UseTrue16WithSramECC in {
+ def : DSReadPat_t16<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
+ }
+}
+
multiclass DSReadPat_t16<DS_Pseudo inst, ValueType vt, string frag> {
let OtherPredicates = [LDSRequiresM0Init] in {
@@ -974,12 +983,7 @@ multiclass DSReadPat_t16<DS_Pseudo inst, ValueType vt, string frag> {
let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = NotUseRealTrue16Insts in {
def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
}
- let OtherPredicates = [NotLDSRequiresM0Init, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts in {
- def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>;
- }
- let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = UseTrue16WithSramECC in {
- def : DSReadPat_t16<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
- }
+ defm : DSReadPat_True16<inst, vt, frag>;
let OtherPredicates = [HasGDS], True16Predicate = NotUseRealTrue16Insts in {
def : DSReadPat<inst, vt, !cast<PatFrag>(!subst("_local", "_region", frag)#"_m0"),
@@ -1012,13 +1016,14 @@ foreach vt = VReg_64.RegTypes in {
defm : DSReadPat_mc <DS_READ_B64, vt, "atomic_load_local">;
}
-defm : DSReadPat_t16 <DS_READ_U8, i16, "atomic_load_aext_8_local">;
+defm : DSReadPat_True16 <DS_READ_U8, i16, "atomic_load_aext_8_local">;
defm : DSReadPat_mc <DS_READ_U8, i32, "atomic_load_aext_8_local">;
-defm : DSReadPat_t16 <DS_READ_U8, i16, "atomic_load_zext_8_local">;
+defm : DSReadPat_True16 <DS_READ_U8, i16, "atomic_load_zext_8_local">;
defm : DSReadPat_mc <DS_READ_U8, i32, "atomic_load_zext_8_local">;
-defm : DSReadPat_t16 <DS_READ_I8, i16, "atomic_load_sext_8_local">;
+defm : DSReadPat_True16 <DS_READ_I8, i16, "atomic_load_sext_8_local">;
defm : DSReadPat_mc <DS_READ_I8, i32, "atomic_load_sext_8_local">;
-defm : DSReadPat_t16 <DS_READ_U16, i16, "atomic_load_nonext_16_local">;
+foreach vt = Reg16Types.types in
+defm : DSReadPat_True16 <DS_READ_U16, vt, "atomic_load_local">;
defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_aext_16_local">;
defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_zext_16_local">;
defm : DSReadPat_mc <DS_READ_I16, i32, "atomic_load_sext_16_local">;
@@ -1066,29 +1071,32 @@ multiclass DSWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
}
}
+multiclass DSWritePat_True16 <DS_Pseudo inst, ValueType vt, string frag> {
+ let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = UseRealTrue16Insts in {
+ def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>;
+ }
+
+ defvar region_frag = !cast<PatFrag>(!subst("_local", "_region", frag)#"_m0");
+ let OtherPredicates = [HasGDS], True16Predicate = UseRealTrue16Insts in {
+ def : DSWritePat_t16<inst, vt, region_frag, /*gds=*/1>;
+ }
+}
+
multiclass DSWritePat_t16 <DS_Pseudo inst, ValueType vt, string frag> {
let OtherPredicates = [LDSRequiresM0Init] in {
def : DSWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
}
- let OtherPredicates = [NotLDSRequiresM0Init] in {
- let True16Predicate = NotUseRealTrue16Insts in {
- def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
- }
- let True16Predicate = UseRealTrue16Insts in {
- def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>;
- }
+ let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = NotUseRealTrue16Insts in {
+ def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
}
defvar region_frag = !cast<PatFrag>(!subst("_local", "_region", frag)#"_m0");
- let OtherPredicates = [HasGDS] in {
- let True16Predicate = NotUseRealTrue16Insts in {
- def : DSWritePat<inst, vt, region_frag, /*gds=*/1>;
- }
- let True16Predicate = UseRealTrue16Insts in {
- def : DSWritePat_t16<inst, vt, region_frag, /*gds=*/1>;
- }
+ let OtherPredicates = [HasGDS], True16Predicate = NotUseRealTrue16Insts in {
+ def : DSWritePat<inst, vt, region_frag, /*gds=*/1>;
}
+
+ defm : DSWritePat_True16<inst, vt, frag>;
}
defm : DSWritePat_mc <DS_WRITE_B8, i32, "truncstorei8_local">;
@@ -1105,9 +1113,10 @@ foreach vt = VReg_64.RegTypes in {
defm : DSWritePat_mc <DS_WRITE_B64, vt, "atomic_store_local">;
}
-defm : DSWritePat_t16 <DS_WRITE_B8, i16, "atomic_store_8_local">;
+defm : DSWritePat_True16 <DS_WRITE_B8, i16, "atomic_store_8_local">;
defm : DSWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
-defm : DSWritePat_t16 <DS_WRITE_B16, i16, "atomic_store_16_local">;
+foreach vt = Reg16Types.types in
+defm : DSWritePat_True16 <DS_WRITE_B16, vt, "atomic_store_local">;
defm : DSWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
let OtherPredicates = [HasD16LoadStore] in {
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index e6c7618a37a64..585e5619d51a5 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -2052,7 +2052,6 @@ defm : FlatLoadPats <FLAT_LOAD_UBYTE, atomic_load_aext_8_flat, i32>;
defm : FlatLoadPats <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i32>;
defm : FlatLoadPats <FLAT_LOAD_USHORT, atomic_load_aext_16_flat, i32>;
defm : FlatLoadPats <FLAT_LOAD_USHORT, atomic_load_zext_16_flat, i32>;
-defm : FlatLoadPats <FLAT_LOAD_USHORT, atomic_load_zext_16_flat, i16>;
defm : FlatLoadPats <FLAT_LOAD_UBYTE, extloadi8_flat, i32>;
defm : FlatLoadPats <FLAT_LOAD_UBYTE, zextloadi8_flat, i32>;
defm : FlatLoadPats <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>;
@@ -2068,14 +2067,8 @@ let True16Predicate = NotUseRealTrue16Insts in {
defm : FlatLoadPats <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>;
defm : FlatLoadPats <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>;
defm : FlatLoadPats <FLAT_LOAD_USHORT, load_flat, i16>;
- defm : FlatLoadPats <FLAT_LOAD_UBYTE, atomic_load_aext_8_flat, i16>;
- defm : FlatLoadPats <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i16>;
- defm : FlatLoadPats <FLAT_LOAD_USHORT, atomic_load_nonext_16_flat, i16>;
- defm : FlatLoadPats <FLAT_LOAD_SBYTE, atomic_load_sext_8_flat, i16>;
defm : FlatStorePats <FLAT_STORE_BYTE, truncstorei8_flat, i16>;
defm : FlatStorePats <FLAT_STORE_SHORT, store_flat, i16>;
- defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i16>;
- defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i16>;
}
let True16Predicate = UseTrue16WithSramECC in {
@@ -2085,7 +2078,8 @@ let True16Predicate = UseTrue16WithSramECC in {
defm : FlatLoadPats_t16 <FLAT_LOAD_USHORT, load_flat, i16>;
defm : FlatLoadPats_t16 <FLAT_LOAD_UBYTE, atomic_load_aext_8_flat, i16>;
defm : FlatLoadPats_t16 <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i16>;
- defm : FlatLoadPats_t16 <FLAT_LOAD_USHORT, atomic_load_nonext_16_flat, i16>;
+ foreach vt = Reg16Types.types in
+ defm : FlatLoadPats_t16 <FLAT_LOAD_USHORT, atomic_load_flat, vt>;
defm : FlatLoadPats_t16 <FLAT_LOAD_SBYTE, atomic_load_sext_8_flat, i16>;
}
@@ -2096,7 +2090,8 @@ let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predi
defm : FlatLoadPats_D16_t16<FLAT_LOAD_SHORT_D16_t16, load_flat, i16>;
defm : FlatLoadPats_D16_t16<FLAT_LOAD_UBYTE_D16_t16, atomic_load_aext_8_flat, i16>;
defm : FlatLoadPats_D16_t16<FLAT_LOAD_UBYTE_D16_t16, atomic_load_zext_8_flat, i16>;
- defm : FlatLoadPats_D16_t16<FLAT_LOAD_SHORT_D16_t16, atomic_load_nonext_16_flat, i16>;
+ foreach vt = Reg16Types.types in
+ defm : FlatLoadPats_D16_t16<FLAT_LOAD_SHORT_D16_t16, atomic_load_flat, vt>;
defm : FlatLoadPats_D16_t16<FLAT_LOAD_SBYTE_D16_t16, atomic_load_sext_8_flat, i16>;
} // End let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predicate = UseRealTrue16Insts
@@ -2104,7 +2099,8 @@ let OtherPredicates = [HasFlatAddressSpace], True16Predicate = UseRealTrue16Inst
defm : FlatStorePats_t16 <FLAT_STORE_BYTE, truncstorei8_flat, i16>;
defm : FlatStorePats_t16 <FLAT_STORE_SHORT, store_flat, i16>;
defm : FlatStorePats_t16 <FLAT_STORE_BYTE, atomic_store_8_flat, i16>;
- defm : FlatStorePats_t16 <FLAT_STORE_SHORT, atomic_store_16_flat, i16>;
+ foreach vt = Reg16Types.types in
+ defm : FlatStorePats_t16 <FLAT_STORE_SHORT, atomic_store_flat, vt>;
}
defm : FlatStorePats <FLAT_STORE_BYTE, truncstorei8_flat, i32>;
@@ -2229,11 +2225,6 @@ defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, atomic_load_sext_16_global, i32>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i32>;
defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, load_global, i16>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_aext_8_global, i16>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_zext_8_global, i16>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, atomic_load_sext_8_global, i16>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_nonext_16_global, i16>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i16>;
}
let True16Predicate = UseTrue16WithSramECC in {
@@ -2246,7 +2237,8 @@ defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, load_global, i16>;
defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_UBYTE, atomic_load_aext_8_global, i16>;
defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_UBYTE, atomic_load_zext_8_global, i16>;
defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_SBYTE, atomic_load_sext_8_global, i16>;
-defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, atomic_load_nonext_16_global, i16>;
+foreach vt = Reg16Types.types in
+defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, atomic_load_global, vt>;
defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i16>;
}
@@ -2258,12 +2250,14 @@ defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", load_global, i16>;
defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", atomic_load_aext_8_global, i16>;
defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", atomic_load_zext_8_global, i16>;
defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SBYTE_D16", atomic_load_sext_8_global, i16>;
-defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", atomic_load_nonext_16_global, i16>;
+foreach vt = Reg16Types.types in
+defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", atomic_load_global, vt>;
defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", atomic_load_zext_16_global, i16>;
defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_BYTE", truncstorei8_global, i16>;
defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", store_global, i16>;
defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_BYTE", atomic_store_8_global, i16>;
-defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", atomic_store_16_global, i16>;
+foreach vt = Reg16Types.types in
+defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", atomic_store_global, vt>;
} // end OtherPredicates = [HasFlatGlobalInsts, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts
// There is no distinction for atomic load lowering during selection;
@@ -2299,15 +2293,14 @@ defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>;
let OtherPredicates = [HasFlatGlobalInsts], True16Predicate = NotUseRealTrue16Insts in {
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i16>;
defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, store_global, i16>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i16>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i16>;
}
let OtherPredicates = [HasFlatGlobalInsts], True16Predicate = UseRealTrue16Insts in {
defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_BYTE", truncstorei8_global, i16>;
defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_SHORT", store_global, i16>;
defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_BYTE", atomic_store_8_global, i16>;
-defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_SHORT", atomic_store_16_global, i16>;
+foreach vt = Reg16Types.types in
+defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_SHORT", atomic_store_global, vt>;
}
let OtherPredicates = [HasD16LoadStore] in {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index fa7c49528a820..d743a5d06adf5 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -659,6 +659,14 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::STORE, MVT::bf16, Promote);
AddPromotedToType(ISD::STORE, MVT::bf16, MVT::i16);
+ if (!Subtarget->useRealTrue16Insts()) {
+ for (MVT VT : {MVT::i16, MVT::f16, MVT::bf16}) {
+ setOperationAction({ISD::ATOMIC_LOAD, ISD::ATOMIC_STORE}, VT, Promote);
+ AddPromotedToType(ISD::ATOMIC_LOAD, VT, MVT::i32);
+ AddPromotedToType(ISD::ATOMIC_STORE, VT, MVT::i32);
+ }
+ }
+
// F16 - VOP1 Actions.
setOperationAction({ISD::FP_ROUND, ISD::STRICT_FP_ROUND, ISD::FCOS,
ISD::FSIN, ISD::FROUND},
diff --git a/llvm/test/CodeGen/AMDGPU/gds-load-store.ll b/llvm/test/CodeGen/AMDGPU/gds-load-store.ll
index f47811c8ee4e9..3fb0f4596a162 100644
--- a/llvm/test/CodeGen/AMDGPU/gds-load-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/gds-load-store.ll
@@ -1521,7 +1521,6 @@ define i32 @atomic_load_zext_i8(ptr addrspace(2) %ptr) {
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_waitcnt expcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_zext_i8:
@@ -1534,7 +1533,6 @@ define i32 @atomic_load_zext_i8(ptr addrspace(2) %ptr) {
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_waitcnt expcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: atomic_load_zext_i8:
@@ -1560,7 +1558,6 @@ define i32 @atomic_load_zext_i8(ptr addrspace(2) %ptr) {
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_waitcnt expcnt(0)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%load = load atomic i8, ptr addrspace(2) %ptr seq_cst, align 1
%ext = zext i8 %load to i32
@@ -1662,7 +1659,6 @@ define i32 @atomic_load_zext_i16(ptr addrspace(2) %ptr) {
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_waitcnt expcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_zext_i16:
@@ -1675,7 +1671,6 @@ define i32 @atomic_load_zext_i16(ptr addrspace(2) %ptr) {
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_waitcnt expcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: atomic_load_zext_i16:
@@ -1701,7 +1696,6 @@ define i32 @atomic_load_zext_i16(ptr addrspace(2) %ptr) {
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_waitcnt expcnt(0)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%load = load atomic i16, ptr addrspace(2) %ptr seq_cst, align 2
%ext = zext i16 %load to i32
More information about the llvm-branch-commits
mailing list