[llvm-branch-commits] [llvm] [AtomicExpand][NVPTX] Support native elementwise atomicrmw (PR #233742)
Yonah Goldberg via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Oct 21 21:09:42 PDT 2026
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/233742
>From c5be83f18deced86e7962a0c164af4c3a5e9b047 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 20 Oct 2026 19:56:43 +0000
Subject: [PATCH] [AtomicExpand][NVPTX] Support native elementwise atomicrmw
---
llvm/include/llvm/CodeGen/TargetLowering.h | 1 +
llvm/lib/CodeGen/AtomicExpandPass.cpp | 70 +-
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 152 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 227 +-
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 47 +-
.../Target/NVPTX/NVPTXSelectionDAGInfo.cpp | 3 +
llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h | 9 +-
.../NVPTX/atomicrmw-elementwise-sm70.ll | 224 +-
.../NVPTX/atomicrmw-elementwise-sm90.ll | 1981 +++--------------
llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll | 95 +
.../NVPTX/expand-atomic-rmw-elementwise.ll | 4 +-
.../X86/expand-atomic-rmw-elementwise.ll | 23 +-
12 files changed, 908 insertions(+), 1928 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index bdb7ae0586e248..c52947ee322f8b 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -2288,6 +2288,7 @@ class LLVM_ABI TargetLoweringBase {
/// the backend. Atomic operations greater than this size (as well
/// as ones that are not naturally aligned), will be expanded by
/// AtomicExpandPass into an __atomic_* library call.
+ /// For elementwise RMWs, the limit applies to each lane.
unsigned getMaxAtomicSizeInBitsSupported() const {
return MaxAtomicSizeInBitsSupported;
}
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 3e25517408b7cd..54c99cbc933728 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -215,25 +215,24 @@ INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
INITIALIZE_PASS_END(AtomicExpandLegacy, DEBUG_TYPE,
"Expand Atomic instructions", false, false)
-// Helper functions to retrieve the size of atomic instructions.
-static unsigned getAtomicOpSize(LoadInst *LI) {
- const DataLayout &DL = LI->getDataLayout();
- return DL.getTypeStoreSize(LI->getType());
+// Helper functions to retrieve the type accessed by atomic instructions.
+static Type *getAtomicOpType(LoadInst *LI) { return LI->getType(); }
+
+static Type *getAtomicOpType(StoreInst *SI) {
+ return SI->getValueOperand()->getType();
}
-static unsigned getAtomicOpSize(StoreInst *SI) {
- const DataLayout &DL = SI->getDataLayout();
- return DL.getTypeStoreSize(SI->getValueOperand()->getType());
+static Type *getAtomicOpType(AtomicRMWInst *RMWI) {
+ return RMWI->getValOperand()->getType();
}
-static unsigned getAtomicOpSize(AtomicRMWInst *RMWI) {
- const DataLayout &DL = RMWI->getDataLayout();
- return DL.getTypeStoreSize(RMWI->getValOperand()->getType());
+static Type *getAtomicOpType(AtomicCmpXchgInst *CASI) {
+ return CASI->getCompareOperand()->getType();
}
-static unsigned getAtomicOpSize(AtomicCmpXchgInst *CASI) {
- const DataLayout &DL = CASI->getDataLayout();
- return DL.getTypeStoreSize(CASI->getCompareOperand()->getType());
+// The storage size of the full access, including all lanes of a vector.
+template <typename Inst> static unsigned getTypeStoreSize(Inst *I) {
+ return I->getDataLayout().getTypeStoreSize(getAtomicOpType(I));
}
/// Copy metadata that's safe to preserve when widening atomics.
@@ -263,25 +262,32 @@ static void copyMetadataForAtomic(Instruction &Dest,
else if (ID == Ctx.getMDKindID("amdgpu.no.fine.grained.memory"))
Dest.setMetadata(ID, N);
- // Losing atomic.ignore.denormal.mode, but it doesn't matter for current
- // uses.
break;
}
}
}
-template <typename Inst>
-static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
- unsigned Size = getAtomicOpSize(I);
- Align Alignment = I->getAlign();
+static bool atomicSizeSupported(const TargetLowering *TLI, unsigned Size,
+ Align Alignment) {
unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8;
return Size <= MaxSize && TLI->isAtomicAlignmentSupported(Alignment, Size);
}
+template <typename Inst>
+static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
+ Type *Ty = getAtomicOpType(I);
+ // Elementwise RMWs require atomic support for each lane. Expansions that
+ // access the whole vector atomically validate that access separately.
+ if (auto *RMWI = dyn_cast<AtomicRMWInst>(I); RMWI && RMWI->isElementwise())
+ Ty = Ty->getScalarType();
+ unsigned Size = I->getDataLayout().getTypeStoreSize(Ty);
+ return atomicSizeSupported(TLI, Size, I->getAlign());
+}
+
template <typename Inst>
static void writeUnsupportedAtomicSizeReason(const TargetLowering *TLI, Inst *I,
raw_ostream &OS) {
- unsigned Size = getAtomicOpSize(I);
+ unsigned Size = getTypeStoreSize(I);
Align Alignment = I->getAlign();
bool NeedSeparator = false;
@@ -934,7 +940,11 @@ bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
return false;
case TargetLoweringBase::AtomicExpansionKind::LLSC: {
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
- unsigned ValueSize = getAtomicOpSize(AI);
+ unsigned ValueSize = getTypeStoreSize(AI);
+ if (!atomicSizeSupported(TLI, ValueSize, AI->getAlign())) {
+ expandAtomicRMWToLibcall(AI);
+ return true;
+ }
if (ValueSize < MinCASSize) {
expandPartwordAtomicRMW(AI,
TargetLoweringBase::AtomicExpansionKind::LLSC);
@@ -950,7 +960,11 @@ bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
}
case TargetLoweringBase::AtomicExpansionKind::CmpXChg: {
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
- unsigned ValueSize = getAtomicOpSize(AI);
+ unsigned ValueSize = getTypeStoreSize(AI);
+ if (!atomicSizeSupported(TLI, ValueSize, AI->getAlign())) {
+ expandAtomicRMWToLibcall(AI);
+ return true;
+ }
if (ValueSize < MinCASSize) {
expandPartwordAtomicRMW(AI,
TargetLoweringBase::AtomicExpansionKind::CmpXChg);
@@ -973,7 +987,7 @@ bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
}
case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic: {
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
- unsigned ValueSize = getAtomicOpSize(AI);
+ unsigned ValueSize = getTypeStoreSize(AI);
if (ValueSize < MinCASSize) {
AtomicRMWInst::BinOp Op = AI->getOperation();
// Widen And/Or/Xor and give the target another chance at expanding it.
@@ -2005,7 +2019,7 @@ Value *AtomicExpandImpl::insertRMWCmpXchgLoop(
bool AtomicExpandImpl::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
- unsigned ValueSize = getAtomicOpSize(CI);
+ unsigned ValueSize = getTypeStoreSize(CI);
switch (TLI->shouldExpandAtomicCmpXchgInIR(CI)) {
default:
@@ -2075,7 +2089,7 @@ void AtomicExpandImpl::expandAtomicLoadToLibcall(LoadInst *I) {
static const RTLIB::Libcall Libcalls[6] = {
RTLIB::ATOMIC_LOAD, RTLIB::ATOMIC_LOAD_1, RTLIB::ATOMIC_LOAD_2,
RTLIB::ATOMIC_LOAD_4, RTLIB::ATOMIC_LOAD_8, RTLIB::ATOMIC_LOAD_16};
- unsigned Size = getAtomicOpSize(I);
+ unsigned Size = getTypeStoreSize(I);
bool Expanded = expandAtomicOpToLibcall(
I, Size, I->getAlign(), I->getPointerOperand(), nullptr, nullptr,
@@ -2088,7 +2102,7 @@ void AtomicExpandImpl::expandAtomicStoreToLibcall(StoreInst *I) {
static const RTLIB::Libcall Libcalls[6] = {
RTLIB::ATOMIC_STORE, RTLIB::ATOMIC_STORE_1, RTLIB::ATOMIC_STORE_2,
RTLIB::ATOMIC_STORE_4, RTLIB::ATOMIC_STORE_8, RTLIB::ATOMIC_STORE_16};
- unsigned Size = getAtomicOpSize(I);
+ unsigned Size = getTypeStoreSize(I);
bool Expanded = expandAtomicOpToLibcall(
I, Size, I->getAlign(), I->getPointerOperand(), I->getValueOperand(),
@@ -2104,7 +2118,7 @@ void AtomicExpandImpl::expandAtomicCASToLibcall(AtomicCmpXchgInst *I,
RTLIB::ATOMIC_COMPARE_EXCHANGE, RTLIB::ATOMIC_COMPARE_EXCHANGE_1,
RTLIB::ATOMIC_COMPARE_EXCHANGE_2, RTLIB::ATOMIC_COMPARE_EXCHANGE_4,
RTLIB::ATOMIC_COMPARE_EXCHANGE_8, RTLIB::ATOMIC_COMPARE_EXCHANGE_16};
- unsigned Size = getAtomicOpSize(I);
+ unsigned Size = getTypeStoreSize(I);
bool Expanded = expandAtomicOpToLibcall(
I, Size, I->getAlign(), I->getPointerOperand(), I->getNewValOperand(),
@@ -2186,7 +2200,7 @@ static ArrayRef<RTLIB::Libcall> GetRMWLibcall(AtomicRMWInst::BinOp Op) {
void AtomicExpandImpl::expandAtomicRMWToLibcall(AtomicRMWInst *I) {
ArrayRef<RTLIB::Libcall> Libcalls = GetRMWLibcall(I->getOperation());
- unsigned Size = getAtomicOpSize(I);
+ unsigned Size = getTypeStoreSize(I);
bool Success = false;
if (!Libcalls.empty())
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index aff6c017d6f74f..8646cb65664279 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -78,7 +78,13 @@ struct NVPTXMemCacheHintAccess {
NVPTX::AddressSpace AddrSpace;
unsigned NumElts;
unsigned EltWidth;
- bool IsVolatile;
+ // Ordering of the selected PTX instruction, which may drop IR volatility.
+ NVPTX::Ordering Ordering;
+
+ bool isVolatileOrMMIO() const {
+ return Ordering == NVPTX::Ordering::Volatile ||
+ Ordering == NVPTX::Ordering::RelaxedMMIO;
+ }
};
struct NVPTXMemCacheHintOperands {
@@ -136,6 +142,7 @@ class NVPTXDAGToDAGISel : public SelectionDAGISel {
void SelectTcgen05Ld(SDNode *N, bool hasOffset = false);
void SelectTcgen05St(SDNode *N, bool hasOffset = false);
void selectAtomicSwap128(SDNode *N);
+ void selectVectorAtomicRMW(SDNode *N);
inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) {
return CurDAG->getTargetConstant(Imm, DL, MVT::i32);
@@ -318,6 +325,11 @@ void NVPTXDAGToDAGISel::Select(SDNode *N) {
case NVPTXISD::ATOMIC_SWAP_B128:
selectAtomicSwap128(N);
return;
+ case NVPTXISD::ATOM_VEC_V1:
+ case NVPTXISD::ATOM_VEC_V2:
+ case NVPTXISD::ATOM_VEC_V4:
+ selectVectorAtomicRMW(N);
+ return;
case ISD::FADD:
case ISD::FMUL:
case ISD::FSUB:
@@ -1256,13 +1268,17 @@ static bool isL1EvictionSupported(const NVPTXSubtarget &Subtarget,
if (Eviction == NVPTX::L1Eviction::Normal)
return true;
- return isLdOrSt(Access) && !Access.IsVolatile &&
+ return isLdOrSt(Access) && !Access.isVolatileOrMMIO() &&
Subtarget.hasL1EvictionHint();
}
static bool isL2PrefetchSupported(const NVPTXSubtarget &Subtarget,
NVPTX::L2Prefetch Prefetch,
NVPTXMemCacheHintAccess Access) {
+ if (Prefetch != NVPTX::L2Prefetch::None &&
+ Access.Ordering == NVPTX::Ordering::RelaxedMMIO)
+ return false;
+
switch (Prefetch) {
case NVPTX::L2Prefetch::None:
return true;
@@ -1285,7 +1301,7 @@ static bool isL2EvictionSupported(const NVPTXSubtarget &Subtarget,
if (Eviction == NVPTX::L2Eviction::Normal)
return true;
- return isLdOrSt(Access) && !Access.IsVolatile &&
+ return isLdOrSt(Access) && !Access.isVolatileOrMMIO() &&
Subtarget.hasL2EvictionHint() && isGlobalOrGeneric(Access.AddrSpace) &&
((Access.NumElts == 8 && Access.EltWidth == 32) ||
(Access.NumElts == 4 && Access.EltWidth == 64));
@@ -1293,7 +1309,7 @@ static bool isL2EvictionSupported(const NVPTXSubtarget &Subtarget,
static bool isCachePolicySupported(const NVPTXSubtarget &Subtarget,
NVPTXMemCacheHintAccess Access) {
- return !Access.IsVolatile && isGlobalOrGeneric(Access.AddrSpace) &&
+ return !Access.isVolatileOrMMIO() && isGlobalOrGeneric(Access.AddrSpace) &&
Subtarget.hasL2CacheHint();
}
@@ -1420,7 +1436,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
LD,
{NVPTXMemCacheHintInstruction::Ld, CodeAddrSpace,
- /*NumElts=*/1, /*EltWidth=*/FromTypeWidth, LD->isVolatile()},
+ /*NumElts=*/1, /*EltWidth=*/FromTypeWidth, Ordering},
DL);
// Create the machine instruction DAG
@@ -1501,7 +1517,7 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
getMemCacheHintOperands(LD,
{NVPTXMemCacheHintInstruction::Ld, CodeAddrSpace,
/*NumElts=*/LD->getNumValues() - 1,
- /*EltWidth=*/FromTypeWidth, LD->isVolatile()},
+ /*EltWidth=*/FromTypeWidth, Ordering},
DL);
const auto [Base, Offset] = selectADDR(N->getOperand(1), CurDAG);
SDValue Ops[] = {getI32Imm(Ordering, DL),
@@ -1570,7 +1586,7 @@ bool NVPTXDAGToDAGISel::tryLDG(MemSDNode *LD) {
const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
LD,
{NVPTXMemCacheHintInstruction::Ld, NVPTX::AddressSpace::Global,
- LD->getNumValues() - 1, FromTypeWidth, LD->isVolatile()},
+ LD->getNumValues() - 1, FromTypeWidth, NVPTX::Ordering::NotAtomic},
DL);
SDValue Ops[] = {getI32Imm(FromType, DL),
getI32Imm(FromTypeWidth, DL),
@@ -1692,7 +1708,7 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
ST,
{NVPTXMemCacheHintInstruction::St, CodeAddrSpace,
- /*NumElts=*/1, /*EltWidth=*/ToTypeWidth, ST->isVolatile()},
+ /*NumElts=*/1, /*EltWidth=*/ToTypeWidth, Ordering},
DL);
SDValue Ops[] = {selectPossiblyImm(Value),
@@ -1753,7 +1769,7 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
ST,
{NVPTXMemCacheHintInstruction::St, CodeAddrSpace,
- /*NumElts=*/NumElts, /*EltWidth=*/ToTypeWidth, ST->isVolatile()},
+ /*NumElts=*/NumElts, /*EltWidth=*/ToTypeWidth, Ordering},
DL);
const auto [Base, Offset] = selectADDR(Addr, CurDAG);
@@ -2375,6 +2391,122 @@ bool NVPTXDAGToDAGISel::tryIntrinsicVoid(SDNode *N) {
}
}
+void NVPTXDAGToDAGISel::selectVectorAtomicRMW(SDNode *N) {
+ MemSDNode *MN = cast<MemSDNode>(N);
+ SDLoc DL(N);
+
+ unsigned NumRegs;
+ switch (N->getOpcode()) {
+ case NVPTXISD::ATOM_VEC_V1:
+ NumRegs = 1;
+ break;
+ case NVPTXISD::ATOM_VEC_V2:
+ NumRegs = 2;
+ break;
+ case NVPTXISD::ATOM_VEC_V4:
+ NumRegs = 4;
+ break;
+ default:
+ llvm_unreachable("Unexpected opcode");
+ }
+
+ // Node layout: chain, ptr, val0..valN-1, op_code
+ const auto [Base, Offset] = selectADDR(N->getOperand(1), CurDAG);
+ unsigned OpCode =
+ cast<ConstantSDNode>(N->getOperand(2 + NumRegs))->getZExtValue();
+ MVT RegVT = N->getSimpleValueType(0);
+
+ // Opcode table indexed by [op][register_type][register_count].
+ // op: 0=fadd, 1=fmin, 2=fmax
+ // register_type: 0=f32, 1=f16, 2=bf16, 3=f16x2, 4=bf16x2
+ // register_count: 0=1, 1=2, 2=4
+ // clang-format off
+ static const unsigned OpcTable[3][5][3] = {
+ // fadd
+ {{0, NVPTX::ATOM_ADD_V2_F32, NVPTX::ATOM_ADD_V4_F32},
+ {0, 0, 0},
+ {0, 0, 0},
+ {NVPTX::INT_PTX_ATOM_ADD_F16X2_r, NVPTX::ATOM_ADD_V2_F16X2, NVPTX::ATOM_ADD_V4_F16X2},
+ {NVPTX::INT_PTX_ATOM_ADD_BF16X2_r, NVPTX::ATOM_ADD_V2_BF16X2, NVPTX::ATOM_ADD_V4_BF16X2}},
+ // fmin
+ {{0, 0, 0},
+ {0, NVPTX::ATOM_MIN_V2_F16, 0},
+ {0, NVPTX::ATOM_MIN_V2_BF16, 0},
+ {0, NVPTX::ATOM_MIN_V2_F16X2, NVPTX::ATOM_MIN_V4_F16X2},
+ {0, NVPTX::ATOM_MIN_V2_BF16X2, NVPTX::ATOM_MIN_V4_BF16X2}},
+ // fmax
+ {{0, 0, 0},
+ {0, NVPTX::ATOM_MAX_V2_F16, 0},
+ {0, NVPTX::ATOM_MAX_V2_BF16, 0},
+ {0, NVPTX::ATOM_MAX_V2_F16X2, NVPTX::ATOM_MAX_V4_F16X2},
+ {0, NVPTX::ATOM_MAX_V2_BF16X2, NVPTX::ATOM_MAX_V4_BF16X2}},
+ };
+ // clang-format on
+
+ unsigned TypeIdx;
+ switch (RegVT.SimpleTy) {
+ case MVT::f32:
+ TypeIdx = 0;
+ break;
+ case MVT::f16:
+ TypeIdx = 1;
+ break;
+ case MVT::bf16:
+ TypeIdx = 2;
+ break;
+ case MVT::v2f16:
+ TypeIdx = 3;
+ break;
+ case MVT::v2bf16:
+ TypeIdx = 4;
+ break;
+ default:
+ report_fatal_error("Unsupported register type for vector atom");
+ }
+
+ unsigned WidthIdx;
+ switch (NumRegs) {
+ case 1:
+ WidthIdx = 0;
+ break;
+ case 2:
+ WidthIdx = 1;
+ break;
+ case 4:
+ WidthIdx = 2;
+ break;
+ default:
+ report_fatal_error("Unsupported vector width for vector atom");
+ }
+
+ assert(OpCode < std::size(OpcTable) && "Unexpected atomic operation");
+ unsigned Opc = OpcTable[OpCode][TypeIdx][WidthIdx];
+ if (!Opc)
+ report_fatal_error("Unsupported vector atom combination");
+
+ if (RegVT == MVT::f32 && Subtarget->hasFeature(NVPTX::PTX94) && !useF32FTZ())
+ Opc = NumRegs == 2 ? NVPTX::ATOM_ADD_NOFTZ_V2_F32
+ : NVPTX::ATOM_ADD_NOFTZ_V4_F32;
+
+ SmallVector<SDValue, 12> Ops{Base, Offset};
+ for (unsigned I = 0; I < NumRegs; ++I)
+ Ops.push_back(N->getOperand(2 + I));
+ Ops.push_back(getI32Imm(getMemOrder(MN), DL));
+ Ops.push_back(getI32Imm(getAtomicScope(MN), DL));
+ Ops.push_back(getI32Imm(getAddrSpace(MN), DL));
+ NVPTXMemCacheHintAccess Access{
+ NVPTXMemCacheHintInstruction::Atom, getAddrSpace(MN), NumRegs,
+ unsigned(RegVT.getFixedSizeInBits()), getMemOrder(MN)};
+ const auto [EvictionAndPrefetchHint, CachePolicyReg] =
+ getMemCacheHintOperands(MN, Access, DL);
+ Ops.append(
+ {EvictionAndPrefetchHint, CachePolicyReg, N->getOperand(0)}); // chain
+
+ auto *ATOM = CurDAG->getMachineNode(Opc, DL, N->getVTList(), Ops);
+ CurDAG->setNodeMemRefs(ATOM, MN->getMemOperand());
+ ReplaceNode(N, ATOM);
+}
+
void NVPTXDAGToDAGISel::selectAtomicSwap128(SDNode *N) {
MemSDNode *AN = cast<MemSDNode>(N);
SDLoc dl(N);
@@ -2391,7 +2523,7 @@ void NVPTXDAGToDAGISel::selectAtomicSwap128(SDNode *N) {
unsigned EltWidth = AN->getMemoryVT().getFixedSizeInBits();
NVPTXMemCacheHintAccess Access{NVPTXMemCacheHintInstruction::Atom,
getAddrSpace(AN),
- /*NumElts=*/1, EltWidth, AN->isVolatile()};
+ /*NumElts=*/1, EltWidth, getMemOrder(AN)};
const auto [EvictionAndPrefetchHint, CachePolicyReg] =
getMemCacheHintOperands(AN, Access, dl);
Ops.push_back(EvictionAndPrefetchHint);
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 3f7c2b3e6a1c04..54b1fb84254304 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -1150,6 +1150,13 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
setOperationAction({ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP}, MVT::i128,
Custom);
+ setOperationAction({ISD::ATOMIC_LOAD_FADD, ISD::ATOMIC_LOAD_FMIN,
+ ISD::ATOMIC_LOAD_FMAX, ISD::ATOMIC_LOAD_FMINIMUMNUM,
+ ISD::ATOMIC_LOAD_FMAXIMUMNUM},
+ {MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v2bf16,
+ MVT::v4bf16, MVT::v8bf16, MVT::v2f32, MVT::v4f32},
+ Custom);
+
// Now deduce the information based on the above mentioned
// actions
computeRegisterProperties(STI.getRegisterInfo());
@@ -3651,9 +3658,106 @@ static SDValue lowerMSTORE(SDValue Op, SelectionDAG &DAG) {
return NewSt;
}
-SDValue
-NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
+enum AtomVecOp : unsigned {
+ ATOM_VEC_FADD = 0,
+ ATOM_VEC_FMIN = 1,
+ ATOM_VEC_FMAX = 2,
+};
+
+// Replace a vector atomic with a target memory node that returns one value
+// per PTX register, followed by the chain. Half and bfloat values use packed
+// pairs when the instruction supports them.
+static void replaceVectorAtomicRMW(SDNode *N, SelectionDAG &DAG,
+ SmallVectorImpl<SDValue> &Results) {
+ auto *AN = cast<AtomicSDNode>(N);
+ EVT VecVT = AN->getValueType(0);
+ EVT EltVT = VecVT.getVectorElementType();
+ unsigned NumElts = VecVT.getVectorNumElements();
+ SDLoc DL(N);
+
+ bool IsAdd = N->getOpcode() == ISD::ATOMIC_LOAD_FADD;
+ // Packed scalar atomics only support add. Two-element min/max use v2.f16
+ // or v2.bf16; wider vectors use packed pairs for all supported operations.
+ bool Pack = EltVT != MVT::f32 && (IsAdd || NumElts > 2);
+ EVT RegVT = Pack ? EVT::getVectorVT(*DAG.getContext(), EltVT, 2) : EltVT;
+ unsigned NumRegs = Pack ? NumElts / 2 : NumElts;
+
+ unsigned NVPTXOpc;
+ switch (NumRegs) {
+ case 1:
+ NVPTXOpc = NVPTXISD::ATOM_VEC_V1;
+ break;
+ case 2:
+ NVPTXOpc = NVPTXISD::ATOM_VEC_V2;
+ break;
+ case 4:
+ NVPTXOpc = NVPTXISD::ATOM_VEC_V4;
+ break;
+ default:
+ llvm_unreachable("Unsupported vector width for atom");
+ }
+
+ unsigned OpCode;
+ switch (N->getOpcode()) {
+ case ISD::ATOMIC_LOAD_FADD:
+ OpCode = ATOM_VEC_FADD;
+ break;
+ case ISD::ATOMIC_LOAD_FMIN:
+ case ISD::ATOMIC_LOAD_FMINIMUMNUM:
+ OpCode = ATOM_VEC_FMIN;
+ break;
+ case ISD::ATOMIC_LOAD_FMAX:
+ case ISD::ATOMIC_LOAD_FMAXIMUMNUM:
+ OpCode = ATOM_VEC_FMAX;
+ break;
+ default:
+ llvm_unreachable("Unsupported atomic op for vector atom");
+ }
+
+ SmallVector<SDValue, 10> Ops;
+ Ops.push_back(AN->getChain());
+ Ops.push_back(AN->getBasePtr());
+ SDValue Val = AN->getVal();
+ for (unsigned I = 0; I < NumRegs; ++I) {
+ if (RegVT == VecVT)
+ Ops.push_back(Val);
+ else
+ Ops.push_back(DAG.getNode(
+ Pack ? ISD::EXTRACT_SUBVECTOR : ISD::EXTRACT_VECTOR_ELT, DL, RegVT,
+ Val, DAG.getVectorIdxConstant(Pack ? 2 * I : I, DL)));
+ }
+ Ops.push_back(DAG.getTargetConstant(OpCode, DL, MVT::i32));
+
+ SmallVector<EVT, 5> ResVTs(NumRegs, RegVT);
+ ResVTs.push_back(MVT::Other);
+
+ SDValue Result = DAG.getMemIntrinsicNode(NVPTXOpc, DL, DAG.getVTList(ResVTs),
+ Ops, VecVT, AN->getMemOperand());
+
+ SmallVector<SDValue, 8> Elts;
+ for (unsigned I = 0; I < NumRegs; ++I)
+ Elts.push_back(Result.getValue(I));
+
+ SDValue Vec = RegVT == VecVT ? Result
+ : DAG.getNode(Pack ? ISD::CONCAT_VECTORS
+ : ISD::BUILD_VECTOR,
+ DL, VecVT, Elts);
+ Results.push_back(Vec);
+ Results.push_back(Result.getValue(NumRegs));
+}
+
+SDValue NVPTXTargetLowering::LowerOperation(SDValue Op,
+ SelectionDAG &DAG) const {
switch (Op.getOpcode()) {
+ case ISD::ATOMIC_LOAD_FADD:
+ case ISD::ATOMIC_LOAD_FMIN:
+ case ISD::ATOMIC_LOAD_FMAX:
+ case ISD::ATOMIC_LOAD_FMINIMUMNUM:
+ case ISD::ATOMIC_LOAD_FMAXIMUMNUM: {
+ SmallVector<SDValue, 2> Results;
+ replaceVectorAtomicRMW(Op.getNode(), DAG, Results);
+ return DAG.getMergeValues(Results, SDLoc(Op));
+ }
case ISD::RETURNADDR:
return SDValue();
case ISD::FRAMEADDR:
@@ -7946,13 +8050,22 @@ void NVPTXTargetLowering::ReplaceNodeResults(
case ISD::ATOMIC_SWAP:
replaceAtomicSwap128(N, DAG, STI, Results);
return;
+ case ISD::ATOMIC_LOAD_FADD:
+ case ISD::ATOMIC_LOAD_FMIN:
+ case ISD::ATOMIC_LOAD_FMAX:
+ case ISD::ATOMIC_LOAD_FMINIMUMNUM:
+ case ISD::ATOMIC_LOAD_FMAXIMUMNUM: {
+ replaceVectorAtomicRMW(N, DAG, Results);
+ return;
+ }
}
}
/// Returns how NVPTX should expand a scalar atomicrmw.
-static TargetLoweringBase::AtomicExpansionKind getScalarAtomicRMWExpansion(
- AtomicRMWInst::BinOp Op, Type *Ty, unsigned AddressSpace, const Function &F,
- bool IgnoreDenormalMode, const NVPTXSubtarget &STI) {
+static TargetLoweringBase::AtomicExpansionKind
+getScalarAtomicRMWExpansion(AtomicRMWInst::BinOp Op, Type *Ty,
+ unsigned AddressSpace, const Function &F,
+ bool IgnoreFTZMismatch, const NVPTXSubtarget &STI) {
using AtomicExpansionKind = TargetLoweringBase::AtomicExpansionKind;
assert(!Ty->isVectorTy() && "Expected a scalar atomicrmw type");
@@ -7972,12 +8085,6 @@ static TargetLoweringBase::AtomicExpansionKind getScalarAtomicRMWExpansion(
? AtomicExpansionKind::Expand
: AtomicExpansionKind::None;
- // Both the -nvptx-allow-ftz-atomics option and per-instruction
- // !atomic.ignore.denormal.mode say that denormal handling is insignificant
- // here, so atom.add may be used even when its FTZ behavior disagrees with
- // the function's.
- const bool IgnoreFTZMismatch = AllowFTZAtomics || IgnoreDenormalMode;
-
if (Ty->isFloatTy()) {
const bool FTZ = F.getDenormalMode(APFloat::IEEEsingle()).Output ==
DenormalMode::PreserveSign;
@@ -8087,15 +8194,73 @@ static TargetLoweringBase::AtomicExpansionKind getScalarAtomicRMWExpansion(
return AtomicExpansionKind::CmpXChg;
}
+// Return the number of lanes in the widest native elementwise atomic for this
+// lane type/operation. Vector atomics require global memory; a generic pointer
+// can also refer to shared memory, where only the packed f16x2/bf16x2 adds are
+// available.
+static unsigned getWidestNativeAtomicRMWVectorWidth(
+ AtomicRMWInst::BinOp Op, Type *Ty, unsigned AddressSpace, const Function &F,
+ bool IgnoreFTZMismatch, const NVPTXSubtarget &STI) {
+ // PTX vector atom only supports floating-point add/min/max. sub is expanded
+ // to add.
+ bool IsAdd = Op == AtomicRMWInst::FAdd || Op == AtomicRMWInst::FSub;
+ if (!IsAdd && Op != AtomicRMWInst::FMin && Op != AtomicRMWInst::FMax &&
+ Op != AtomicRMWInst::FMinimumNum && Op != AtomicRMWInst::FMaximumNum)
+ return 0;
+
+ bool HasVector = AddressSpace == ADDRESS_SPACE_GLOBAL &&
+ STI.hasFeature(NVPTX::SM90) && STI.hasFeature(NVPTX::PTX81);
+ if (Ty->isFloatTy()) {
+ bool FTZ = F.getDenormalMode(APFloat::IEEEsingle()).Output ==
+ DenormalMode::PreserveSign;
+ return IsAdd && HasVector &&
+ (FTZ || IgnoreFTZMismatch || STI.hasFeature(NVPTX::PTX94))
+ ? 4
+ : 0;
+ }
+
+ if (!Ty->isHalfTy() && !Ty->isBFloatTy())
+ return 0;
+
+ // Native f16/bf16 atomics always preserve denormals. If the function requires
+ // f16 flushing, use a CAS loop unless the mismatch is explicitly allowed.
+ if (Ty->isHalfTy()) {
+ bool FTZ = F.getDenormalMode(APFloat::IEEEhalf()).Output ==
+ DenormalMode::PreserveSign;
+ if (FTZ && !IgnoreFTZMismatch)
+ return 0;
+ }
+
+ // Global vector atomics support up to eight f16/bf16 lanes.
+ if (HasVector)
+ return 8;
+
+ // Otherwise, only packed f16x2/bf16x2 add is available. These forms also
+ // support generic and shared addresses.
+ if (!IsAdd)
+ return 0;
+ if (AddressSpace != ADDRESS_SPACE_GENERIC &&
+ AddressSpace != ADDRESS_SPACE_GLOBAL &&
+ AddressSpace != ADDRESS_SPACE_SHARED)
+ return 0;
+
+ bool HasPackedAdd =
+ Ty->isHalfTy()
+ ? STI.hasFeature(NVPTX::SM60) && STI.hasFeature(NVPTX::PTX62)
+ : STI.hasFeature(NVPTX::SM90) && STI.hasFeature(NVPTX::PTX78);
+ return HasPackedAdd ? 2 : 0;
+}
+
NVPTXTargetLowering::AtomicExpansionKind
NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
const DataLayout &DL = AI->getDataLayout();
AtomicRMWInst::BinOp Op = AI->getOperation();
- const bool IgnoreDenormalMode =
+ // atom.add may be used even when its FTZ behavior disagrees with
+ // the functions'.
+ const bool IgnoreFTZMismatch =
+ AllowFTZAtomics ||
AI->hasMetadata(LLVMContext::MD_atomic_ignore_denormal_mode);
- // TODO: once we support native elementwise vector atoms,
- // return `AtomicExpansionKind::None` to preserve and lower them.
if (AI->isElementwise() &&
cast<FixedVectorType>(AI->getType())->getNumElements() > 1) {
auto *VecTy = cast<FixedVectorType>(AI->getType());
@@ -8104,6 +8269,21 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
uint64_t VecBytes = DL.getTypeStoreSize(VecTy).getFixedValue();
uint64_t VecBits = DL.getTypeStoreSizeInBits(VecTy).getFixedValue();
+ unsigned NativeWidth = getWidestNativeAtomicRMWVectorWidth(
+ Op, Ty, AI->getPointerAddressSpace(), *AI->getFunction(),
+ IgnoreFTZMismatch, STI);
+ if (NativeWidth) {
+ // Rewrite subtraction to addition in IR before preserving or splitting
+ // the native vector atomic.
+ if (Op == AtomicRMWInst::FSub)
+ return AtomicExpansionKind::Expand;
+ if (VecTy->getNumElements() <= NativeWidth && AI->getAlign() >= VecBytes)
+ return AtomicExpansionKind::None;
+ // Halving will find a native vector width or a scalar operation with
+ // sufficient alignment.
+ return AtomicExpansionKind::Expand;
+ }
+
if (Op == AtomicRMWInst::BinOp::Xchg) {
// Split exchanges that are too wide or insufficiently aligned until
// each piece can be lowered atomically.
@@ -8121,20 +8301,21 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
// If the scalar lane op is natively supported or expands to one, return
// Expand so halving eventually bottoms out at the scalar base case. The
// latter includes sub and fsub, which expand to add and fadd, respectively.
- AtomicExpansionKind LaneExpansionKind = getScalarAtomicRMWExpansion(
- Op, Ty, AI->getPointerAddressSpace(), *AI->getFunction(),
- IgnoreDenormalMode, STI);
+ AtomicExpansionKind LaneExpansionKind =
+ getScalarAtomicRMWExpansion(Op, Ty, AI->getPointerAddressSpace(),
+ *AI->getFunction(), IgnoreFTZMismatch, STI);
if (LaneExpansionKind == AtomicExpansionKind::None ||
LaneExpansionKind == AtomicExpansionKind::Expand)
return AtomicExpansionKind::Expand;
- // If the whole vector fits a single native cmpxchg, emit one wide
- // cmpxchg loop at the current width.
- if (VecBits <= getMaxAtomicSizeInBitsSupported())
+ // If the whole vector fits a single native cmpxchg and has sufficient
+ // alignment, emit one wide cmpxchg loop at the current width.
+ if (VecBits <= getMaxAtomicSizeInBitsSupported() &&
+ AI->getAlign() >= VecBytes)
return AtomicExpansionKind::CmpXChg;
- // If the vector is too wide for a single native cmpxchg, halve further to
- // emit multiple cmpxchg loops.
+ // Otherwise, halve until each piece is small enough and sufficiently
+ // aligned for a native cmpxchg.
return AtomicExpansionKind::Expand;
}
@@ -8145,7 +8326,7 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
Ty = VecTy->getElementType();
}
return getScalarAtomicRMWExpansion(Op, Ty, AI->getPointerAddressSpace(),
- *AI->getFunction(), IgnoreDenormalMode,
+ *AI->getFunction(), IgnoreFTZMismatch,
STI);
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 2b4071c0357cee..e760444e9dbd9f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -3366,7 +3366,7 @@ class AtomicSDNodeXForm<SDNode atomic> {
unsigned EltWidth = MN->getMemoryVT().getFixedSizeInBits();
NVPTXMemCacheHintAccess Access{
NVPTXMemCacheHintInstruction::Atom, getAddrSpace(MN),
- /*NumElts=*/1, EltWidth, MN->isVolatile()};
+ /*NumElts=*/1, EltWidth, getMemOrder(MN)};
return getMemCacheHintOperands(MN, Access, SDLoc(N),
/*EmitDiagnostics=*/true)
.EvictionAndPrefetchHint;
@@ -3377,7 +3377,7 @@ class AtomicSDNodeXForm<SDNode atomic> {
unsigned EltWidth = MN->getMemoryVT().getFixedSizeInBits();
NVPTXMemCacheHintAccess Access{
NVPTXMemCacheHintInstruction::Atom, getAddrSpace(MN),
- /*NumElts=*/1, EltWidth, MN->isVolatile()};
+ /*NumElts=*/1, EltWidth, getMemOrder(MN)};
return getMemCacheHintOperands(MN, Access, SDLoc(N),
/*EmitDiagnostics=*/false)
.CachePolicyReg;
@@ -3447,9 +3447,52 @@ defm INT_PTX_ATOM_ADD_64 : F_ATOMIC_2<I64RT, atomic_load_add_i64, "add", "u64",
defm INT_PTX_ATOM_ADD_F16 : F_ATOMIC_2<F16RT, atomic_load_fadd, "add.noftz", "f16", atomic_load_fadd, [SM70, PTX63]>;
defm INT_PTX_ATOM_ADD_BF16 : F_ATOMIC_2<BF16RT, atomic_load_fadd, "add.noftz", "bf16", atomic_load_fadd, [SM90]>;
+defm INT_PTX_ATOM_ADD_F16X2 : F_ATOMIC_2<F16X2RT, atomic_load_fadd, "add.noftz", "f16x2", atomic_load_fadd, [SM60, PTX62]>;
+defm INT_PTX_ATOM_ADD_BF16X2 : F_ATOMIC_2<BF16X2RT, atomic_load_fadd, "add.noftz", "bf16x2", atomic_load_fadd, [SM90, PTX78]>;
defm INT_PTX_ATOM_ADD_F32 : F_ATOMIC_2<F32RT, atomic_load_fadd, "add", "f32", atomic_load_fadd>;
defm INT_PTX_ATOM_ADD_F64 : F_ATOMIC_2<F64RT, atomic_load_fadd, "add", "f64", atomic_load_fadd, [hasAtomAddF64]>;
+// Native vector atomics use one register per f32 element or packed pair of
+// f16/bf16 elements. Two-element f16/bf16 min/max use two 16-bit registers.
+class F_ATOMIC_VECTOR<RegTyInfo T, int N, string Op, string Type> :
+ NVPTXInst<
+ !dag(outs, !listsplat(T.RC, N),
+ !foreach(I, !range(N), "dst" # I)),
+ !con((ins ADDR:$addr),
+ !dag(ins, !listsplat(T.RC, N),
+ !foreach(I, !range(N), "src" # I)),
+ (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
+ EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy)),
+ "atom${sem:sem}${scope:scope}${addsp:addsp}." # Op #
+ "${evictionAndPrefetchHint:l2}.v" # N # "." # Type # " \t{{" #
+ !interleave(!foreach(I, !range(N), "$dst" # I), ", ") #
+ "}}, [$addr], {{" #
+ !interleave(!foreach(I, !range(N), "$src" # I), ", ") #
+ "}}${policy};"> {
+ let mayLoad = true;
+ let mayStore = true;
+ let hasSideEffects = true;
+ let Predicates = [SM90, PTX81];
+}
+
+foreach N = [2, 4] in {
+ def ATOM_ADD_V # N # _F32 : F_ATOMIC_VECTOR<F32RT, N, "add", "f32">;
+ let Predicates = [SM90, PTX94] in
+ def ATOM_ADD_NOFTZ_V # N # _F32 :
+ F_ATOMIC_VECTOR<F32RT, N, "add.noftz", "f32">;
+ foreach T = [F16X2RT, BF16X2RT] in {
+ foreach Op = ["add", "min", "max"] in
+ def ATOM_ # !toupper(Op) # _V # N # _ # !toupper(T.PtxType) :
+ F_ATOMIC_VECTOR<T, N, Op # ".noftz", T.PtxType>;
+ }
+}
+foreach T = [F16RT, BF16RT] in {
+ foreach Op = ["min", "max"] in
+ def ATOM_ # !toupper(Op) # _V2_ # !toupper(T.PtxType) :
+ F_ATOMIC_VECTOR<T, 2, Op # ".noftz", T.PtxType>;
+}
+
// atom_swap
defm INT_PTX_ATOM_SWAP_32 : F_ATOMIC_2<I32RT, atomic_swap_i32, "exch", "b32", atomic_swap>;
defm INT_PTX_ATOM_SWAP_64 : F_ATOMIC_2<I64RT, atomic_swap_i64, "exch", "b64", atomic_swap>;
diff --git a/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.cpp
index dac41465e1bddd..9c2d736884322e 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.cpp
@@ -36,6 +36,9 @@ const char *NVPTXSelectionDAGInfo::getTargetNodeName(unsigned Opcode) const {
MAKE_CASE(NVPTXISD::StoreV2)
MAKE_CASE(NVPTXISD::StoreV4)
MAKE_CASE(NVPTXISD::StoreV8)
+ MAKE_CASE(NVPTXISD::ATOM_VEC_V1)
+ MAKE_CASE(NVPTXISD::ATOM_VEC_V2)
+ MAKE_CASE(NVPTXISD::ATOM_VEC_V4)
MAKE_CASE(NVPTXISD::SETP_F16X2)
MAKE_CASE(NVPTXISD::SETP_BF16X2)
MAKE_CASE(NVPTXISD::UNPACK_VECTOR)
diff --git a/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h b/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h
index 9dd0a1eaa5856b..b345570c23cb6c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h
@@ -42,7 +42,14 @@ enum NodeType : unsigned {
StoreV2,
StoreV4,
StoreV8,
- LAST_MEMORY_OPCODE = StoreV8,
+
+ /// Native vector atomicrmw nodes. Each takes (chain, ptr, val0..valN-1,
+ /// op_code) and produces (res0..resN-1, chain), with one value per PTX
+ /// register. op_code is 0=fadd, 1=fmin, or 2=fmax.
+ ATOM_VEC_V1,
+ ATOM_VEC_V2,
+ ATOM_VEC_V4,
+ LAST_MEMORY_OPCODE = ATOM_VEC_V4,
};
} // namespace NVPTXISD
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
index bc8ba880324632..d7a72b76ecf7d7 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
@@ -16224,17 +16224,14 @@ define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-LABEL: fadd_acq_rel_v2f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<3>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2f16_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r2, [%rd1], %r1;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -16243,19 +16240,17 @@ define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM70-LABEL: fadd_acq_rel_v4f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<5>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4f16_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs5, [%rd1], %rs1;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs2;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs3;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs4;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r4, [%rd1+4], %r2;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -16264,7 +16259,6 @@ define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM70-LABEL: fadd_acq_rel_v8f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .b16 %rs<17>;
; SM70-NEXT: .reg .b32 %r<9>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
@@ -16272,22 +16266,10 @@ define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f16_global_cta_param_1];
-; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs9, [%rd1], %rs7;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+2], %rs8;
-; SM70-NEXT: mov.b32 %r5, {%rs9, %rs10};
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+4], %rs5;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs6;
-; SM70-NEXT: mov.b32 %r6, {%rs11, %rs12};
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs13, [%rd1+8], %rs3;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+10], %rs4;
-; SM70-NEXT: mov.b32 %r7, {%rs13, %rs14};
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs1;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs2;
-; SM70-NEXT: mov.b32 %r8, {%rs15, %rs16};
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r8, [%rd1+12], %r4;
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT: ret;
@@ -16354,74 +16336,54 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-DISALLOW: {
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<5>;
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<4>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r2, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM70-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r3;
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-ALLOW: {
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<5>;
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<4>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: neg.f16x2 %r2, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM70-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r3;
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM70-FTZ-DISALLOW: {
-; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<5>;
-; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<4>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r2, %r1;
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM70-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r3;
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM70-FTZ-ALLOW: {
-; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<5>;
-; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<4>;
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r2, %r1;
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM70-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r3;
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -16430,8 +16392,7 @@ define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-DISALLOW: {
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<9>;
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
@@ -16440,20 +16401,15 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r3, %r2;
; SM70-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r4, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r5, [%rd1], %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r6, [%rd1+4], %r3;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.v4.b16 [func_retval0], {%rs3, %rs4, %rs7, %rs8};
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-ALLOW: {
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<9>;
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<7>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
@@ -16462,20 +16418,15 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-NOFTZ-ALLOW-NEXT: neg.f16x2 %r3, %r2;
; SM70-NOFTZ-ALLOW-NEXT: neg.f16x2 %r4, %r1;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r3;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r5, [%rd1], %r4;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r6, [%rd1+4], %r3;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.v4.b16 [func_retval0], {%rs3, %rs4, %rs7, %rs8};
+; SM70-NOFTZ-ALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM70-FTZ-DISALLOW: {
-; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<9>;
-; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
@@ -16484,20 +16435,15 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r3, %r2;
; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r4, %r1;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r3;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r5, [%rd1], %r4;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r6, [%rd1+4], %r3;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.v4.b16 [func_retval0], {%rs3, %rs4, %rs7, %rs8};
+; SM70-FTZ-DISALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM70-FTZ-ALLOW: {
-; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<9>;
-; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<7>;
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
@@ -16506,14 +16452,10 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r3, %r2;
; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r4, %r1;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r3;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r5, [%rd1], %r4;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r6, [%rd1+4], %r3;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.v4.b16 [func_retval0], {%rs3, %rs4, %rs7, %rs8};
+; SM70-FTZ-ALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -16522,7 +16464,6 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-DISALLOW: {
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<17>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
@@ -16534,29 +16475,16 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r7, %r2;
; SM70-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r8, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r8;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r9, {%rs3, %rs4};
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r7;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r10, {%rs7, %rs8};
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r6;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+8], %rs9;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+10], %rs10;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r11, {%rs11, %rs12};
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r5;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs13;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs14;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r12, {%rs15, %rs16};
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r9, [%rd1], %r8;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r10, [%rd1+4], %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r11, [%rd1+8], %r6;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r12, [%rd1+12], %r5;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-ALLOW: {
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<17>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<13>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
@@ -16568,29 +16496,16 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NOFTZ-ALLOW-NEXT: neg.f16x2 %r7, %r2;
; SM70-NOFTZ-ALLOW-NEXT: neg.f16x2 %r8, %r1;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r8;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r9, {%rs3, %rs4};
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r7;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r10, {%rs7, %rs8};
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r6;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+8], %rs9;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+10], %rs10;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r11, {%rs11, %rs12};
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r5;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs13;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs14;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r12, {%rs15, %rs16};
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r9, [%rd1], %r8;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r10, [%rd1+4], %r7;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r11, [%rd1+8], %r6;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r12, [%rd1+12], %r5;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM70-FTZ-DISALLOW: {
-; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<17>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
@@ -16602,29 +16517,16 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r7, %r2;
; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r8, %r1;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r8;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r9, {%rs3, %rs4};
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r7;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r10, {%rs7, %rs8};
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r6;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+8], %rs9;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+10], %rs10;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r11, {%rs11, %rs12};
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r5;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs13;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs14;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r12, {%rs15, %rs16};
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r9, [%rd1], %r8;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r10, [%rd1+4], %r7;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r11, [%rd1+8], %r6;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r12, [%rd1+12], %r5;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM70-FTZ-ALLOW: {
-; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<17>;
; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<13>;
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
@@ -16636,22 +16538,10 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r7, %r2;
; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r8, %r1;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r8;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r9, {%rs3, %rs4};
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r7;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r10, {%rs7, %rs8};
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r6;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+8], %rs9;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+10], %rs10;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r11, {%rs11, %rs12};
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r5;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs13;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs14;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r12, {%rs15, %rs16};
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r9, [%rd1], %r8;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r10, [%rd1+4], %r7;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r11, [%rd1+8], %r6;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r12, [%rd1+12], %r5;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-ALLOW-NEXT: ret;
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
index 258c08ec88b99c..bfad103a0c05b4 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
@@ -10235,11 +10235,8 @@ define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v2.f32 {%r3, %r4}, [%rd1], {%r1, %r2};
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -10250,11 +10247,8 @@ define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.v2.f32 {%r3, %r4}, [%rd1], {%r1, %r2};
; SM90-FTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-FTZ-DISALLOW-NEXT: ret;
;
@@ -10265,11 +10259,8 @@ define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v2.f32 {%r3, %r4}, [%rd1], {%r1, %r2};
; SM90-FTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
@@ -10348,13 +10339,8 @@ define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v4.f32 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -10365,13 +10351,8 @@ define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.v4.f32 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-FTZ-DISALLOW-NEXT: ret;
;
@@ -10382,13 +10363,8 @@ define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v4.f32 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
@@ -10522,14 +10498,8 @@ define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r5, %r6, %r7, %r8};
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r13, %r14, %r15, %r16}, [%rd1+16], {%r1, %r2, %r3, %r4};
; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -10545,14 +10515,8 @@ define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r5, %r6, %r7, %r8};
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r13, %r14, %r15, %r16}, [%rd1+16], {%r1, %r2, %r3, %r4};
; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -10568,14 +10532,8 @@ define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r5, %r6, %r7, %r8};
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r13, %r14, %r15, %r16}, [%rd1+16], {%r1, %r2, %r3, %r4};
; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -10696,10 +10654,7 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r3, %r2;
; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r4, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r4;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r3;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v2.f32 {%r5, %r6}, [%rd1], {%r4, %r3};
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -10713,10 +10668,7 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r3, %r2;
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r4, %r1;
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r4;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r3;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.v2.f32 {%r5, %r6}, [%rd1], {%r4, %r3};
; SM90-FTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-FTZ-DISALLOW-NEXT: ret;
;
@@ -10730,10 +10682,7 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r3, %r2;
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r4, %r1;
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r4;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r3;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v2.f32 {%r5, %r6}, [%rd1], {%r4, %r3};
; SM90-FTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
@@ -10817,12 +10766,7 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r6, %r3;
; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r7, %r2;
; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r8, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r8;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r7;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r6;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r5;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -10838,12 +10782,7 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r6, %r3;
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r7, %r2;
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r8, %r1;
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r8;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r7;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r6;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r5;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-FTZ-DISALLOW-NEXT: ret;
;
@@ -10859,12 +10798,7 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r6, %r3;
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r7, %r2;
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r8, %r1;
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r8;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r7;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r6;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r5;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
@@ -11014,14 +10948,8 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r15, %r10;
; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r16, %r9;
; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r17, [%rd1], %r16;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r18, [%rd1+4], %r15;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r19, [%rd1+8], %r14;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r20, [%rd1+12], %r13;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r21, [%rd1+16], %r8;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r22, [%rd1+20], %r7;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r23, [%rd1+24], %r6;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r5;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r17, %r18, %r19, %r20}, [%rd1], {%r16, %r15, %r14, %r13};
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r21, %r22, %r23, %r24}, [%rd1+16], {%r8, %r7, %r6, %r5};
; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
@@ -11045,14 +10973,8 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r15, %r10;
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r16, %r9;
; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r17, [%rd1], %r16;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r18, [%rd1+4], %r15;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r19, [%rd1+8], %r14;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r20, [%rd1+12], %r13;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r21, [%rd1+16], %r8;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r22, [%rd1+20], %r7;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r23, [%rd1+24], %r6;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r5;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r17, %r18, %r19, %r20}, [%rd1], {%r16, %r15, %r14, %r13};
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r21, %r22, %r23, %r24}, [%rd1+16], {%r8, %r7, %r6, %r5};
; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
@@ -11076,14 +10998,8 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r15, %r10;
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r16, %r9;
; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r17, [%rd1], %r16;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r18, [%rd1+4], %r15;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r19, [%rd1+8], %r14;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r20, [%rd1+12], %r13;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r21, [%rd1+16], %r8;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r22, [%rd1+20], %r7;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r23, [%rd1+24], %r6;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r5;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r17, %r18, %r19, %r20}, [%rd1], {%r16, %r15, %r14, %r13};
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r21, %r22, %r23, %r24}, [%rd1+16], {%r8, %r7, %r6, %r5};
; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
@@ -16416,17 +16332,14 @@ define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM90-LABEL: fadd_acq_rel_v2f16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<3>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2f16_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r2, [%rd1], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -16435,19 +16348,14 @@ define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM90-LABEL: fadd_acq_rel_v4f16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<5>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4f16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4f16_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs5, [%rd1], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs2;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs3;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs4;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.v2.f16x2 {%r3, %r4}, [%rd1], {%r1, %r2};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -16456,31 +16364,13 @@ define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM90-LABEL: fadd_acq_rel_v8f16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<17>;
; SM90-NEXT: .reg .b32 %r<9>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v8f16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f16_global_cta_param_1];
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs9, [%rd1], %rs7;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+2], %rs8;
-; SM90-NEXT: mov.b32 %r5, {%rs9, %rs10};
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+4], %rs5;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs6;
-; SM90-NEXT: mov.b32 %r6, {%rs11, %rs12};
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs13, [%rd1+8], %rs3;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+10], %rs4;
-; SM90-NEXT: mov.b32 %r7, {%rs13, %rs14};
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs2;
-; SM90-NEXT: mov.b32 %r8, {%rs15, %rs16};
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.v4.f16x2 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
@@ -16546,74 +16436,54 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM90-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<5>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<4>;
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r2, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<5>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<4>;
; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: neg.f16x2 %r2, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
; SM90-FTZ-DISALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<5>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<4>;
; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r2, %r1;
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-FTZ-DISALLOW-NEXT: ret;
;
; SM90-FTZ-ALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<5>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<4>;
; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r2, %r1;
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -16622,8 +16492,7 @@ define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM90-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<9>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
@@ -16631,21 +16500,13 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r3, %r2;
; SM90-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v4.b16 [func_retval0], {%rs3, %rs4, %rs7, %rs8};
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v2.f16x2 {%r5, %r6}, [%rd1], {%r4, %r3};
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<9>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<7>;
; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
@@ -16653,21 +16514,13 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: neg.f16x2 %r3, %r2;
; SM90-NOFTZ-ALLOW-NEXT: neg.f16x2 %r4, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b16 [func_retval0], {%rs3, %rs4, %rs7, %rs8};
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v2.f16x2 {%r5, %r6}, [%rd1], {%r4, %r3};
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
; SM90-FTZ-DISALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<9>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
@@ -16675,21 +16528,13 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r3, %r2;
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r4, %r1;
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r3;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b16 [func_retval0], {%rs3, %rs4, %rs7, %rs8};
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v2.f16x2 {%r5, %r6}, [%rd1], {%r4, %r3};
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-FTZ-DISALLOW-NEXT: ret;
;
; SM90-FTZ-ALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<9>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<7>;
; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
@@ -16697,15 +16542,8 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r3, %r2;
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r4, %r1;
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r3;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b16 [func_retval0], {%rs3, %rs4, %rs7, %rs8};
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v2.f16x2 {%r5, %r6}, [%rd1], {%r4, %r3};
+; SM90-FTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -16714,7 +16552,6 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM90-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<17>;
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
@@ -16725,30 +16562,12 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r6, %r3;
; SM90-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r7, %r2;
; SM90-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r8, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r8;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r9, {%rs3, %rs4};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r7;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r10, {%rs7, %rs8};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r6;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+8], %rs9;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+10], %rs10;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r11, {%rs11, %rs12};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r5;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs13;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs14;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r12, {%rs15, %rs16};
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v4.f16x2 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<17>;
; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<13>;
; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-ALLOW-EMPTY:
@@ -16759,30 +16578,12 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NOFTZ-ALLOW-NEXT: neg.f16x2 %r6, %r3;
; SM90-NOFTZ-ALLOW-NEXT: neg.f16x2 %r7, %r2;
; SM90-NOFTZ-ALLOW-NEXT: neg.f16x2 %r8, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r8;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r9, {%rs3, %rs4};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r7;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r10, {%rs7, %rs8};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r6;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+8], %rs9;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+10], %rs10;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r11, {%rs11, %rs12};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r5;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs13;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs14;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r12, {%rs15, %rs16};
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v4.f16x2 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
; SM90-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<17>;
; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-DISALLOW-EMPTY:
@@ -16793,30 +16594,12 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r6, %r3;
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r7, %r2;
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r8, %r1;
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r8;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r9, {%rs3, %rs4};
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r7;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r10, {%rs7, %rs8};
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r6;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+8], %rs9;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+10], %rs10;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r11, {%rs11, %rs12};
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r5;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs13;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs14;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r12, {%rs15, %rs16};
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v4.f16x2 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-FTZ-DISALLOW-NEXT: ret;
;
; SM90-FTZ-ALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<17>;
; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<13>;
; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-ALLOW-EMPTY:
@@ -16827,24 +16610,7 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r6, %r3;
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r7, %r2;
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r8, %r1;
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r8;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r9, {%rs3, %rs4};
-; SM90-FTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r7;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs5;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs6;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r10, {%rs7, %rs8};
-; SM90-FTZ-ALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r6;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+8], %rs9;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+10], %rs10;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r11, {%rs11, %rs12};
-; SM90-FTZ-ALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r5;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs13;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs14;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r12, {%rs15, %rs16};
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v4.f16x2 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
@@ -17004,1153 +16770,249 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
}
define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v2f16_global_cta(
+; SM90-LABEL: fmin_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fmin_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.min.noftz.v2.f16 {%rs3, %rs4}, [%rd1], {%rs1, %rs2};
+; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fmin_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.min.noftz.v2.f16x2 {%r3, %r4}, [%rd1], {%r1, %r2};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fmin_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.min.noftz.v4.f16x2 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
; SM90-NOFTZ-DISALLOW: {
; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<14>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r6, 65535, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r6;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r3, %r4, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB297_1;
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r7, %r13, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM90-NOFTZ-DISALLOW-NEXT: max.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: or.b32 %r11, %r10, %r9;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r13, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB300_1;
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r12, %r3, %r1;
; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM90-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v2f16_global_cta(
+; SM90-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
; SM90-NOFTZ-ALLOW: {
; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<14>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM90-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r6, 65535, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r6;
+; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-NOFTZ-ALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r3, %r4, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB297_1;
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r7, %r13, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM90-NOFTZ-ALLOW-NEXT: max.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: or.b32 %r11, %r10, %r9;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r13, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB300_1;
; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r12, %r3, %r1;
; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
-; SM90-FTZ-DISALLOW-LABEL: fmin_acq_rel_v2f16_global_cta(
+; SM90-FTZ-DISALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
; SM90-FTZ-DISALLOW: {
; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<14>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-FTZ-DISALLOW-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r6, 65535, %r1;
+; SM90-FTZ-DISALLOW-NEXT: not.b32 %r2, %r6;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r3, %r4, %r2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB297_1;
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r7, %r13, %r1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-FTZ-DISALLOW-NEXT: or.b32 %r11, %r10, %r9;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r13, %r3;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB300_1;
; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r12, %r3, %r1;
; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
; SM90-FTZ-DISALLOW-NEXT: ret;
;
-; SM90-FTZ-ALLOW-LABEL: fmin_acq_rel_v2f16_global_cta(
+; SM90-FTZ-ALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
; SM90-FTZ-ALLOW: {
; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<14>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-FTZ-ALLOW-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM90-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r6, 65535, %r1;
+; SM90-FTZ-ALLOW-NEXT: not.b32 %r2, %r6;
+; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-FTZ-ALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r3, %r4, %r2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB297_1;
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r7, %r13, %r1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM90-FTZ-ALLOW-NEXT: max.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-FTZ-ALLOW-NEXT: or.b32 %r11, %r10, %r9;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r13, %r3;
+; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB300_1;
; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r12, %r3, %r1;
; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; SM90-FTZ-ALLOW-NEXT: ret;
- %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
- ret <2 x half> %retval
-}
-
-define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v4f16_global_cta(
-; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-NOFTZ-DISALLOW-EMPTY:
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB298_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r3, %r5, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r4, %r6, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB298_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-NOFTZ-DISALLOW-NEXT: ret;
-;
-; SM90-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v4f16_global_cta(
-; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-NOFTZ-ALLOW-EMPTY:
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB298_1: // %atomicrmw.start
-; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r3, %r5, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r4, %r6, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB298_1;
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-NOFTZ-ALLOW-NEXT: ret;
-;
-; SM90-FTZ-DISALLOW-LABEL: fmin_acq_rel_v4f16_global_cta(
-; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-FTZ-DISALLOW-EMPTY:
-; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: $L__BB298_1: // %atomicrmw.start
-; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r3, %r5, %r1;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r4, %r6, %r2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB298_1;
-; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-FTZ-DISALLOW-NEXT: ret;
-;
-; SM90-FTZ-ALLOW-LABEL: fmin_acq_rel_v4f16_global_cta(
-; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-FTZ-ALLOW-EMPTY:
-; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-FTZ-ALLOW-NEXT: $L__BB298_1: // %atomicrmw.start
-; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r3, %r5, %r1;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r4, %r6, %r2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB298_1;
-; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-FTZ-ALLOW-NEXT: ret;
- %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
- ret <4 x half> %retval
-}
-
-define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v8f16_global_cta(
-; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<25>;
-; SM90-NOFTZ-DISALLOW-EMPTY:
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 cmp, {0, 0};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 swap, {0, 0};
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 {%rd2, %rd3}, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: }
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd3;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd3;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB299_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r5, %r9, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r5;
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r6, %r10, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd5, %r6;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd6, %rd5, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd7, %rd4, %rd6;
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r7, %r11, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r7;
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r8, %r12, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd9, %r8;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd10, %rd9, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd11, %rd8, %rd10;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd12, %r9;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r10;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd14, %rd13, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd15, %rd12, %rd14;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd16, %r11;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r12;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd18, %rd17, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd19, %rd16, %rd18;
-; SM90-NOFTZ-DISALLOW-NEXT: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 cmp, {%rd15, %rd19};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 swap, {%rd7, %rd11};
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 {%rd20, %rd21}, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: }
-; SM90-NOFTZ-DISALLOW-NEXT: xor.b64 %rd22, %rd21, %rd19;
-; SM90-NOFTZ-DISALLOW-NEXT: xor.b64 %rd23, %rd20, %rd15;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd24, %rd23, %rd22;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd21;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd20;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd20;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd24, 0;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB299_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-NOFTZ-DISALLOW-NEXT: ret;
-;
-; SM90-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v8f16_global_cta(
-; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<25>;
-; SM90-NOFTZ-ALLOW-EMPTY:
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 cmp, {0, 0};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 swap, {0, 0};
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 {%rd2, %rd3}, dst;
-; SM90-NOFTZ-ALLOW-NEXT: }
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd3;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd3;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB299_1: // %atomicrmw.start
-; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r5, %r9, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r5;
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r6, %r10, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd5, %r6;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd6, %rd5, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd7, %rd4, %rd6;
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r7, %r11, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r7;
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r8, %r12, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd9, %r8;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd10, %rd9, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd11, %rd8, %rd10;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd12, %r9;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r10;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd14, %rd13, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd15, %rd12, %rd14;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd16, %r11;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r12;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd18, %rd17, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd19, %rd16, %rd18;
-; SM90-NOFTZ-ALLOW-NEXT: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 cmp, {%rd15, %rd19};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 swap, {%rd7, %rd11};
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 {%rd20, %rd21}, dst;
-; SM90-NOFTZ-ALLOW-NEXT: }
-; SM90-NOFTZ-ALLOW-NEXT: xor.b64 %rd22, %rd21, %rd19;
-; SM90-NOFTZ-ALLOW-NEXT: xor.b64 %rd23, %rd20, %rd15;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd24, %rd23, %rd22;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd21;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd20;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd20;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd24, 0;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB299_1;
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-NOFTZ-ALLOW-NEXT: ret;
-;
-; SM90-FTZ-DISALLOW-LABEL: fmin_acq_rel_v8f16_global_cta(
-; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<25>;
-; SM90-FTZ-DISALLOW-EMPTY:
-; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 cmp, {0, 0};
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 swap, {0, 0};
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 {%rd2, %rd3}, dst;
-; SM90-FTZ-DISALLOW-NEXT: }
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd3;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd3;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: $L__BB299_1: // %atomicrmw.start
-; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r5, %r9, %r1;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r5;
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r6, %r10, %r2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd5, %r6;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd6, %rd5, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd7, %rd4, %rd6;
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r7, %r11, %r3;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r7;
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r8, %r12, %r4;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd9, %r8;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd10, %rd9, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd11, %rd8, %rd10;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd12, %r9;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r10;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd14, %rd13, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd15, %rd12, %rd14;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd16, %r11;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r12;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd18, %rd17, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd19, %rd16, %rd18;
-; SM90-FTZ-DISALLOW-NEXT: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 cmp, {%rd15, %rd19};
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 swap, {%rd7, %rd11};
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 {%rd20, %rd21}, dst;
-; SM90-FTZ-DISALLOW-NEXT: }
-; SM90-FTZ-DISALLOW-NEXT: xor.b64 %rd22, %rd21, %rd19;
-; SM90-FTZ-DISALLOW-NEXT: xor.b64 %rd23, %rd20, %rd15;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd24, %rd23, %rd22;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd21;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd20;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd20;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd24, 0;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB299_1;
-; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-FTZ-DISALLOW-NEXT: ret;
-;
-; SM90-FTZ-ALLOW-LABEL: fmin_acq_rel_v8f16_global_cta(
-; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<25>;
-; SM90-FTZ-ALLOW-EMPTY:
-; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 cmp, {0, 0};
-; SM90-FTZ-ALLOW-NEXT: mov.b128 swap, {0, 0};
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 {%rd2, %rd3}, dst;
-; SM90-FTZ-ALLOW-NEXT: }
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd3;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd3;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM90-FTZ-ALLOW-NEXT: $L__BB299_1: // %atomicrmw.start
-; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r5, %r9, %r1;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r5;
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r6, %r10, %r2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd5, %r6;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd6, %rd5, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd7, %rd4, %rd6;
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r7, %r11, %r3;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r7;
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r8, %r12, %r4;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd9, %r8;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd10, %rd9, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd11, %rd8, %rd10;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd12, %r9;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r10;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd14, %rd13, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd15, %rd12, %rd14;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd16, %r11;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r12;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd18, %rd17, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd19, %rd16, %rd18;
-; SM90-FTZ-ALLOW-NEXT: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 cmp, {%rd15, %rd19};
-; SM90-FTZ-ALLOW-NEXT: mov.b128 swap, {%rd7, %rd11};
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 {%rd20, %rd21}, dst;
-; SM90-FTZ-ALLOW-NEXT: }
-; SM90-FTZ-ALLOW-NEXT: xor.b64 %rd22, %rd21, %rd19;
-; SM90-FTZ-ALLOW-NEXT: xor.b64 %rd23, %rd20, %rd15;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd24, %rd23, %rd22;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd21;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd20;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd20;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd24, 0;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB299_1;
-; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-FTZ-ALLOW-NEXT: ret;
- %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
- ret <8 x half> %retval
-}
-
-define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
-; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<14>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
-; SM90-NOFTZ-DISALLOW-EMPTY:
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r6, 65535, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r6;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r7, %r13, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16 %rs3, %rs2, %rs1;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r10, %r13, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b32 %r11, %r10, %r9;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r13, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB300_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r12, %r3, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
-; SM90-NOFTZ-DISALLOW-NEXT: ret;
-;
-; SM90-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
-; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<14>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
-; SM90-NOFTZ-ALLOW-EMPTY:
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r6, 65535, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r6;
-; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
-; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r7, %r13, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
-; SM90-NOFTZ-ALLOW-NEXT: max.f16 %rs3, %rs2, %rs1;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r10, %r13, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: or.b32 %r11, %r10, %r9;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r13, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB300_1;
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r12, %r3, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
-; SM90-NOFTZ-ALLOW-NEXT: ret;
-;
-; SM90-FTZ-DISALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
-; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<14>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
-; SM90-FTZ-DISALLOW-EMPTY:
-; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
-; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r6, 65535, %r1;
-; SM90-FTZ-DISALLOW-NEXT: not.b32 %r2, %r6;
-; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
-; SM90-FTZ-DISALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
-; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r7, %r13, %r1;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16 %rs3, %rs2, %rs1;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
-; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, %r1;
-; SM90-FTZ-DISALLOW-NEXT: and.b32 %r10, %r13, %r2;
-; SM90-FTZ-DISALLOW-NEXT: or.b32 %r11, %r10, %r9;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r13, %r3;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB300_1;
-; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r12, %r3, %r1;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
-; SM90-FTZ-DISALLOW-NEXT: ret;
-;
-; SM90-FTZ-ALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
-; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<14>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
-; SM90-FTZ-ALLOW-EMPTY:
-; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
-; SM90-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-FTZ-ALLOW-NEXT: shl.b32 %r6, 65535, %r1;
-; SM90-FTZ-ALLOW-NEXT: not.b32 %r2, %r6;
-; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
-; SM90-FTZ-ALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
-; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: shr.u32 %r7, %r13, %r1;
-; SM90-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16 %rs3, %rs2, %rs1;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
-; SM90-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, %r1;
-; SM90-FTZ-ALLOW-NEXT: and.b32 %r10, %r13, %r2;
-; SM90-FTZ-ALLOW-NEXT: or.b32 %r11, %r10, %r9;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r13, %r3;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB300_1;
-; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-ALLOW-NEXT: shr.u32 %r12, %r3, %r1;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
-; SM90-FTZ-ALLOW-NEXT: ret;
- %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
- ret <1 x half> %retval
-}
-
-define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v2f16_global_cta(
-; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-DISALLOW-EMPTY:
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB301_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r3, %r4, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB301_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: ret;
-;
-; SM90-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v2f16_global_cta(
-; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-ALLOW-EMPTY:
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB301_1: // %atomicrmw.start
-; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r3, %r4, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB301_1;
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; SM90-NOFTZ-ALLOW-NEXT: ret;
-;
-; SM90-FTZ-DISALLOW-LABEL: fmax_acq_rel_v2f16_global_cta(
-; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-DISALLOW-EMPTY:
-; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-FTZ-DISALLOW-NEXT: $L__BB301_1: // %atomicrmw.start
-; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r3, %r4, %r2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB301_1;
-; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; SM90-FTZ-DISALLOW-NEXT: ret;
-;
-; SM90-FTZ-ALLOW-LABEL: fmax_acq_rel_v2f16_global_cta(
-; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-ALLOW-EMPTY:
-; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-FTZ-ALLOW-NEXT: $L__BB301_1: // %atomicrmw.start
-; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r3, %r4, %r2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB301_1;
-; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; SM90-FTZ-ALLOW-NEXT: ret;
- %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
- ret <2 x half> %retval
-}
-
-define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v4f16_global_cta(
-; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-NOFTZ-DISALLOW-EMPTY:
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB302_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r3, %r5, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r4, %r6, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB302_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-NOFTZ-DISALLOW-NEXT: ret;
-;
-; SM90-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v4f16_global_cta(
-; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-NOFTZ-ALLOW-EMPTY:
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB302_1: // %atomicrmw.start
-; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r3, %r5, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r4, %r6, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB302_1;
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-NOFTZ-ALLOW-NEXT: ret;
-;
-; SM90-FTZ-DISALLOW-LABEL: fmax_acq_rel_v4f16_global_cta(
-; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-FTZ-DISALLOW-EMPTY:
-; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: $L__BB302_1: // %atomicrmw.start
-; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r3, %r5, %r1;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r4, %r6, %r2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB302_1;
-; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-FTZ-DISALLOW-NEXT: ret;
-;
-; SM90-FTZ-ALLOW-LABEL: fmax_acq_rel_v4f16_global_cta(
-; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-FTZ-ALLOW-EMPTY:
-; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-FTZ-ALLOW-NEXT: $L__BB302_1: // %atomicrmw.start
-; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r3, %r5, %r1;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r4, %r6, %r2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB302_1;
-; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-FTZ-ALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
; SM90-FTZ-ALLOW-NEXT: ret;
- %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
- ret <4 x half> %retval
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
}
-define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v8f16_global_cta(
-; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<25>;
-; SM90-NOFTZ-DISALLOW-EMPTY:
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 cmp, {0, 0};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 swap, {0, 0};
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 {%rd2, %rd3}, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: }
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd3;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd3;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB303_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r5, %r9, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r5;
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r6, %r10, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd5, %r6;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd6, %rd5, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd7, %rd4, %rd6;
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r7, %r11, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r7;
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r8, %r12, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd9, %r8;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd10, %rd9, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd11, %rd8, %rd10;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd12, %r9;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r10;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd14, %rd13, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd15, %rd12, %rd14;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd16, %r11;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r12;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd18, %rd17, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd19, %rd16, %rd18;
-; SM90-NOFTZ-DISALLOW-NEXT: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 cmp, {%rd15, %rd19};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 swap, {%rd7, %rd11};
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 {%rd20, %rd21}, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: }
-; SM90-NOFTZ-DISALLOW-NEXT: xor.b64 %rd22, %rd21, %rd19;
-; SM90-NOFTZ-DISALLOW-NEXT: xor.b64 %rd23, %rd20, %rd15;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd24, %rd23, %rd22;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd21;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd20;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd20;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd24, 0;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB303_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-NOFTZ-DISALLOW-NEXT: ret;
-;
-; SM90-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v8f16_global_cta(
-; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<25>;
-; SM90-NOFTZ-ALLOW-EMPTY:
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 cmp, {0, 0};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 swap, {0, 0};
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 {%rd2, %rd3}, dst;
-; SM90-NOFTZ-ALLOW-NEXT: }
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd3;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd3;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB303_1: // %atomicrmw.start
-; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r5, %r9, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r5;
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r6, %r10, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd5, %r6;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd6, %rd5, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd7, %rd4, %rd6;
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r7, %r11, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r7;
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r8, %r12, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd9, %r8;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd10, %rd9, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd11, %rd8, %rd10;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd12, %r9;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r10;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd14, %rd13, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd15, %rd12, %rd14;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd16, %r11;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r12;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd18, %rd17, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd19, %rd16, %rd18;
-; SM90-NOFTZ-ALLOW-NEXT: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 cmp, {%rd15, %rd19};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 swap, {%rd7, %rd11};
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 {%rd20, %rd21}, dst;
-; SM90-NOFTZ-ALLOW-NEXT: }
-; SM90-NOFTZ-ALLOW-NEXT: xor.b64 %rd22, %rd21, %rd19;
-; SM90-NOFTZ-ALLOW-NEXT: xor.b64 %rd23, %rd20, %rd15;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd24, %rd23, %rd22;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd21;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd20;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd20;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd24, 0;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB303_1;
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-NOFTZ-ALLOW-NEXT: ret;
-;
-; SM90-FTZ-DISALLOW-LABEL: fmax_acq_rel_v8f16_global_cta(
-; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<25>;
-; SM90-FTZ-DISALLOW-EMPTY:
-; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 cmp, {0, 0};
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 swap, {0, 0};
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 {%rd2, %rd3}, dst;
-; SM90-FTZ-DISALLOW-NEXT: }
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd3;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd3;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: $L__BB303_1: // %atomicrmw.start
-; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r5, %r9, %r1;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r5;
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r6, %r10, %r2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd5, %r6;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd6, %rd5, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd7, %rd4, %rd6;
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r7, %r11, %r3;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r7;
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r8, %r12, %r4;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd9, %r8;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd10, %rd9, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd11, %rd8, %rd10;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd12, %r9;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r10;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd14, %rd13, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd15, %rd12, %rd14;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd16, %r11;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r12;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd18, %rd17, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd19, %rd16, %rd18;
-; SM90-FTZ-DISALLOW-NEXT: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 cmp, {%rd15, %rd19};
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 swap, {%rd7, %rd11};
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 {%rd20, %rd21}, dst;
-; SM90-FTZ-DISALLOW-NEXT: }
-; SM90-FTZ-DISALLOW-NEXT: xor.b64 %rd22, %rd21, %rd19;
-; SM90-FTZ-DISALLOW-NEXT: xor.b64 %rd23, %rd20, %rd15;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd24, %rd23, %rd22;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd21;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd20;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd20;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd24, 0;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB303_1;
-; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-FTZ-DISALLOW-NEXT: ret;
-;
-; SM90-FTZ-ALLOW-LABEL: fmax_acq_rel_v8f16_global_cta(
-; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<25>;
-; SM90-FTZ-ALLOW-EMPTY:
-; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 cmp, {0, 0};
-; SM90-FTZ-ALLOW-NEXT: mov.b128 swap, {0, 0};
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 {%rd2, %rd3}, dst;
-; SM90-FTZ-ALLOW-NEXT: }
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd3;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd3;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM90-FTZ-ALLOW-NEXT: $L__BB303_1: // %atomicrmw.start
-; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r5, %r9, %r1;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r5;
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r6, %r10, %r2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd5, %r6;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd6, %rd5, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd7, %rd4, %rd6;
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r7, %r11, %r3;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r7;
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r8, %r12, %r4;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd9, %r8;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd10, %rd9, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd11, %rd8, %rd10;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd12, %r9;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r10;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd14, %rd13, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd15, %rd12, %rd14;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd16, %r11;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r12;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd18, %rd17, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd19, %rd16, %rd18;
-; SM90-FTZ-ALLOW-NEXT: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 cmp, {%rd15, %rd19};
-; SM90-FTZ-ALLOW-NEXT: mov.b128 swap, {%rd7, %rd11};
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 {%rd20, %rd21}, dst;
-; SM90-FTZ-ALLOW-NEXT: }
-; SM90-FTZ-ALLOW-NEXT: xor.b64 %rd22, %rd21, %rd19;
-; SM90-FTZ-ALLOW-NEXT: xor.b64 %rd23, %rd20, %rd15;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd24, %rd23, %rd22;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd21;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd20;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd20;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd24, 0;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB303_1;
-; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-FTZ-ALLOW-NEXT: ret;
+define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fmax_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.max.noftz.v2.f16 {%rs3, %rs4}, [%rd1], {%rs1, %rs2};
+; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.max.noftz.v2.f16x2 {%r3, %r4}, [%rd1], {%r1, %r2};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.max.noftz.v4.f16x2 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
ret <8 x half> %retval
}
@@ -19478,17 +18340,14 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fadd_acq_rel_v2bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<3>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2bf16_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs3, [%rd1], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs4, [%rd1+2], %rs2;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.bf16x2 %r2, [%rd1], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -19497,19 +18356,14 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM90-LABEL: fadd_acq_rel_v4bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<5>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4bf16_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs5, [%rd1], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs6, [%rd1+2], %rs2;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs7, [%rd1+4], %rs3;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs8, [%rd1+6], %rs4;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.v2.bf16x2 {%r3, %r4}, [%rd1], {%r1, %r2};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -19518,31 +18372,13 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM90-LABEL: fadd_acq_rel_v8bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<17>;
; SM90-NEXT: .reg .b32 %r<9>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs9, [%rd1], %rs7;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs10, [%rd1+2], %rs8;
-; SM90-NEXT: mov.b32 %r5, {%rs9, %rs10};
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs11, [%rd1+4], %rs5;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs12, [%rd1+6], %rs6;
-; SM90-NEXT: mov.b32 %r6, {%rs11, %rs12};
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs13, [%rd1+8], %rs3;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs14, [%rd1+10], %rs4;
-; SM90-NEXT: mov.b32 %r7, {%rs13, %rs14};
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs15, [%rd1+12], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs16, [%rd1+14], %rs2;
-; SM90-NEXT: mov.b32 %r8, {%rs15, %rs16};
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.v4.bf16x2 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
@@ -19569,20 +18405,15 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fsub_acq_rel_v2bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<5>;
-; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b32 %r<4>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
; SM90-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v2bf16_global_cta_param_1];
; SM90-NEXT: neg.bf16x2 %r2, %r1;
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs3, [%rd1], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs4, [%rd1+2], %rs2;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.bf16x2 %r3, [%rd1], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -19591,8 +18422,7 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM90-LABEL: fsub_acq_rel_v4bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<9>;
-; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b32 %r<7>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
@@ -19600,15 +18430,8 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
; SM90-NEXT: neg.bf16x2 %r3, %r2;
; SM90-NEXT: neg.bf16x2 %r4, %r1;
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs3, [%rd1], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs4, [%rd1+2], %rs2;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r3;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs7, [%rd1+4], %rs5;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs8, [%rd1+6], %rs6;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v4.b16 [func_retval0], {%rs3, %rs4, %rs7, %rs8};
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.v2.bf16x2 {%r5, %r6}, [%rd1], {%r4, %r3};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -19617,7 +18440,6 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM90-LABEL: fsub_acq_rel_v8bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<17>;
; SM90-NEXT: .reg .b32 %r<13>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
@@ -19628,24 +18450,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: neg.bf16x2 %r6, %r3;
; SM90-NEXT: neg.bf16x2 %r7, %r2;
; SM90-NEXT: neg.bf16x2 %r8, %r1;
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r8;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs3, [%rd1], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs4, [%rd1+2], %rs2;
-; SM90-NEXT: mov.b32 %r9, {%rs3, %rs4};
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r7;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs7, [%rd1+4], %rs5;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs8, [%rd1+6], %rs6;
-; SM90-NEXT: mov.b32 %r10, {%rs7, %rs8};
-; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r6;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs11, [%rd1+8], %rs9;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs12, [%rd1+10], %rs10;
-; SM90-NEXT: mov.b32 %r11, {%rs11, %rs12};
-; SM90-NEXT: mov.b32 {%rs13, %rs14}, %r5;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs15, [%rd1+12], %rs13;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs16, [%rd1+14], %rs14;
-; SM90-NEXT: mov.b32 %r12, {%rs15, %rs16};
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.v4.bf16x2 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
@@ -19696,25 +18501,14 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fmin_acq_rel_v2bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b16 %rs<5>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB321_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: min.bf16x2 %r3, %r4, %r2;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB321_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fmin_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.min.noftz.v2.bf16 {%rs3, %rs4}, [%rd1], {%rs1, %rs2};
+; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -19723,37 +18517,14 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM90-LABEL: fmin_acq_rel_v4bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<7>;
-; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB322_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: min.bf16x2 %r3, %r5, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-NEXT: min.bf16x2 %r4, %r6, %r2;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB322_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.min.noftz.v2.bf16x2 {%r3, %r4}, [%rd1], {%r1, %r2};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -19762,66 +18533,14 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM90-LABEL: fmin_acq_rel_v8bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<13>;
-; SM90-NEXT: .reg .b64 %rd<25>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {0, 0};
-; SM90-NEXT: mov.b128 swap, {0, 0};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd2, %rd3}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: mov.b64 {_, %r12}, %rd3;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd3;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT: $L__BB323_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: min.bf16x2 %r5, %r9, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r5;
-; SM90-NEXT: min.bf16x2 %r6, %r10, %r2;
-; SM90-NEXT: cvt.u64.u32 %rd5, %r6;
-; SM90-NEXT: shl.b64 %rd6, %rd5, 32;
-; SM90-NEXT: or.b64 %rd7, %rd4, %rd6;
-; SM90-NEXT: min.bf16x2 %r7, %r11, %r3;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r7;
-; SM90-NEXT: min.bf16x2 %r8, %r12, %r4;
-; SM90-NEXT: cvt.u64.u32 %rd9, %r8;
-; SM90-NEXT: shl.b64 %rd10, %rd9, 32;
-; SM90-NEXT: or.b64 %rd11, %rd8, %rd10;
-; SM90-NEXT: cvt.u64.u32 %rd12, %r9;
-; SM90-NEXT: cvt.u64.u32 %rd13, %r10;
-; SM90-NEXT: shl.b64 %rd14, %rd13, 32;
-; SM90-NEXT: or.b64 %rd15, %rd12, %rd14;
-; SM90-NEXT: cvt.u64.u32 %rd16, %r11;
-; SM90-NEXT: cvt.u64.u32 %rd17, %r12;
-; SM90-NEXT: shl.b64 %rd18, %rd17, 32;
-; SM90-NEXT: or.b64 %rd19, %rd16, %rd18;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd15, %rd19};
-; SM90-NEXT: mov.b128 swap, {%rd7, %rd11};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd20, %rd21}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd22, %rd21, %rd19;
-; SM90-NEXT: xor.b64 %rd23, %rd20, %rd15;
-; SM90-NEXT: or.b64 %rd24, %rd23, %rd22;
-; SM90-NEXT: mov.b64 {_, %r12}, %rd21;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd20;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd20;
-; SM90-NEXT: setp.ne.b64 %p1, %rd24, 0;
-; SM90-NEXT: @%p1 bra $L__BB323_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.min.noftz.v4.bf16x2 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -19871,25 +18590,14 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fmax_acq_rel_v2bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b16 %rs<5>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB325_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: max.bf16x2 %r3, %r4, %r2;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB325_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fmax_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.max.noftz.v2.bf16 {%rs3, %rs4}, [%rd1], {%rs1, %rs2};
+; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -19898,37 +18606,14 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM90-LABEL: fmax_acq_rel_v4bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<7>;
-; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB326_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: max.bf16x2 %r3, %r5, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-NEXT: max.bf16x2 %r4, %r6, %r2;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB326_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.max.noftz.v2.bf16x2 {%r3, %r4}, [%rd1], {%r1, %r2};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -19937,66 +18622,14 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM90-LABEL: fmax_acq_rel_v8bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<13>;
-; SM90-NEXT: .reg .b64 %rd<25>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {0, 0};
-; SM90-NEXT: mov.b128 swap, {0, 0};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd2, %rd3}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: mov.b64 {_, %r12}, %rd3;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd3;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT: $L__BB327_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: max.bf16x2 %r5, %r9, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r5;
-; SM90-NEXT: max.bf16x2 %r6, %r10, %r2;
-; SM90-NEXT: cvt.u64.u32 %rd5, %r6;
-; SM90-NEXT: shl.b64 %rd6, %rd5, 32;
-; SM90-NEXT: or.b64 %rd7, %rd4, %rd6;
-; SM90-NEXT: max.bf16x2 %r7, %r11, %r3;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r7;
-; SM90-NEXT: max.bf16x2 %r8, %r12, %r4;
-; SM90-NEXT: cvt.u64.u32 %rd9, %r8;
-; SM90-NEXT: shl.b64 %rd10, %rd9, 32;
-; SM90-NEXT: or.b64 %rd11, %rd8, %rd10;
-; SM90-NEXT: cvt.u64.u32 %rd12, %r9;
-; SM90-NEXT: cvt.u64.u32 %rd13, %r10;
-; SM90-NEXT: shl.b64 %rd14, %rd13, 32;
-; SM90-NEXT: or.b64 %rd15, %rd12, %rd14;
-; SM90-NEXT: cvt.u64.u32 %rd16, %r11;
-; SM90-NEXT: cvt.u64.u32 %rd17, %r12;
-; SM90-NEXT: shl.b64 %rd18, %rd17, 32;
-; SM90-NEXT: or.b64 %rd19, %rd16, %rd18;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd15, %rd19};
-; SM90-NEXT: mov.b128 swap, {%rd7, %rd11};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd20, %rd21}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd22, %rd21, %rd19;
-; SM90-NEXT: xor.b64 %rd23, %rd20, %rd15;
-; SM90-NEXT: or.b64 %rd24, %rd23, %rd22;
-; SM90-NEXT: mov.b64 {_, %r12}, %rd21;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd20;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd20;
-; SM90-NEXT: setp.ne.b64 %p1, %rd24, 0;
-; SM90-NEXT: @%p1 bra $L__BB327_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.max.noftz.v4.bf16x2 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll b/llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll
index 44f2645c145691..40ee54c2805153 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll
@@ -68,6 +68,101 @@ define i128 @atomicrmw_xchg_l2_cache_policy_b128(ptr addrspace(1) %p, i128 %v) {
ret i128 %old
}
+define <4 x float> @atomicrmw_fadd_v4f32_l2_cache_policy(ptr addrspace(1) %p, <4 x float> %v) {
+; CHECK-LABEL: atomicrmw_fadd_v4f32_l2_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: atom.relaxed.sys.global.add.L2::cache_hint.v4.f32 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4}, %rd2;
+ %old = atomicrmw elementwise fadd ptr addrspace(1) %p, <4 x float> %v monotonic, align 16, !mem.cache_hint !1
+ ret <4 x float> %old
+}
+
+; IR volatility does not restrict cache policies on PTX atom instructions.
+define <4 x float> @atomicrmw_fadd_v4f32_volatile_l2_cache_policy(ptr addrspace(1) %p, <4 x float> %v) {
+; CHECK-LABEL: atomicrmw_fadd_v4f32_volatile_l2_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: atom.relaxed.sys.global.add.L2::cache_hint.v4.f32 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4}, %rd2;
+ %old = atomicrmw volatile elementwise fadd ptr addrspace(1) %p, <4 x float> %v monotonic, align 16, !mem.cache_hint !1
+ ret <4 x float> %old
+}
+
+define <4 x half> @atomicrmw_fmin_v4f16_l2_cache_policy(ptr addrspace(1) %p, <4 x half> %v) {
+; CHECK-LABEL: atomicrmw_fmin_v4f16_l2_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: atom.relaxed.sys.global.min.noftz.L2::cache_hint.v2.f16x2 {%r3, %r4}, [%rd1], {%r1, %r2}, %rd2;
+ %old = atomicrmw elementwise fmin ptr addrspace(1) %p, <4 x half> %v monotonic, align 8, !mem.cache_hint !1
+ ret <4 x half> %old
+}
+
+define <2 x bfloat> @atomicrmw_fmax_v2bf16_l2_cache_policy(ptr addrspace(1) %p, <2 x bfloat> %v) {
+; CHECK-LABEL: atomicrmw_fmax_v2bf16_l2_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: atom.relaxed.sys.global.max.noftz.L2::cache_hint.v2.bf16 {%rs3, %rs4}, [%rd1], {%rs1, %rs2}, %rd2;
+ %old = atomicrmw elementwise fmax ptr addrspace(1) %p, <2 x bfloat> %v monotonic, align 4, !mem.cache_hint !1
+ ret <2 x bfloat> %old
+}
+
+; Splitting an elementwise atomic preserves the cache policy on each piece.
+define <8 x float> @atomicrmw_fadd_v8f32_l2_cache_policy(ptr addrspace(1) %p, <8 x float> %v) {
+; CHECK-LABEL: atomicrmw_fadd_v8f32_l2_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: atom.relaxed.sys.global.add.L2::cache_hint.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r5, %r6, %r7, %r8}, %rd2;
+; CHECK: atom.relaxed.sys.global.add.L2::cache_hint.v4.f32 {%r13, %r14, %r15, %r16}, [%rd1+16], {%r1, %r2, %r3, %r4}, %rd2;
+ %old = atomicrmw elementwise fadd ptr addrspace(1) %p, <8 x float> %v monotonic, align 32, !mem.cache_hint !1
+ ret <8 x float> %old
+}
+
+; Cmpxchg expansion drops cache policies unsupported by atom.cas.
+define <2 x i32> @atomicrmw_nand_v2i32_l2_cache_policy(ptr addrspace(1) %p, <2 x i32> %v) {
+; CHECK-LABEL: atomicrmw_nand_v2i32_l2_cache_policy(
+; CHECK: ld.relaxed.sys.global.b64 %rd5, [%rd2];
+; CHECK: mov.b64 {%r1, %r2}, %rd3;
+; CHECK: mov.b64 {%r3, %r4}, %rd5;
+; CHECK: mov.b64 %rd4, {%r8, %r7};
+; CHECK: atom.relaxed.sys.global.cas.b64 %rd1, [%rd2], %rd5, %rd4;
+; CHECK: mov.b64 %rd5, %rd1;
+ %old = atomicrmw elementwise nand ptr addrspace(1) %p, <2 x i32> %v monotonic, align 8, !mem.cache_hint !1
+ ret <2 x i32> %old
+}
+
+define i32 @atomicrmw_add_volatile_l2_cache_policy(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: atomicrmw_add_volatile_l2_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: atom.relaxed.sys.global.add.L2::cache_hint.u32 %r2, [%rd1], %r1, %rd2;
+ %old = atomicrmw volatile add ptr addrspace(1) %p, i32 %v monotonic, align 4, !mem.cache_hint !1
+ ret i32 %old
+}
+
+; Relaxed volatile loads/stores use MMIO forms, which cannot take cache policies.
+define i32 @atomic_load_store_volatile_l2_cache_policy(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: atomic_load_store_volatile_l2_cache_policy(
+; CHECK: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK: st.mmio.relaxed.sys.global.b32 [%rd1], %r2;
+ %old = load atomic volatile i32, ptr addrspace(1) %p monotonic, align 4, !mem.cache_hint !3
+ store atomic volatile i32 %v, ptr addrspace(1) %p monotonic, align 4, !mem.cache_hint !2
+ ret i32 %old
+}
+
+; Acquire/release forms drop the PTX volatile modifier and can retain hints.
+define i32 @atomic_load_store_volatile_ordered_cache_policy(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: atomic_load_store_volatile_ordered_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.acquire.sys.global.L1::evict_first.L2::cache_hint.L2::128B.b32 %r1, [%rd1], %rd2;
+; CHECK: st.release.sys.global.L2::cache_hint.b32 [%rd1], %r2, %rd2;
+ %old = load atomic volatile i32, ptr addrspace(1) %p acquire, align 4, !mem.cache_hint !3
+ store atomic volatile i32 %v, ptr addrspace(1) %p release, align 4, !mem.cache_hint !2
+ ret i32 %old
+}
+
+; PTX volatile loads support prefetch sizes but not eviction or cache policies.
+define i32 @load_store_volatile_cache_policy(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: load_store_volatile_cache_policy(
+; CHECK: ld.volatile.global.L2::128B.b32 %r1, [%rd1];
+; CHECK: st.volatile.global.b32 [%rd1], %r2;
+ %old = load volatile i32, ptr addrspace(1) %p, align 4, !mem.cache_hint !3
+ store volatile i32 %v, ptr addrspace(1) %p, align 4, !mem.cache_hint !2
+ ret i32 %old
+}
+
!0 = !{i32 0, !10}
!1 = !{i32 0, !11}
!2 = !{i32 1, !11}
diff --git a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
index d5d2f36cad9539..f4c4a4d7cce710 100644
--- a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
+++ b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
@@ -3,7 +3,7 @@
target triple = "nvptx64-nvidia-cuda"
-; A <1 x T> elementwise atomicrmw collapses directly to a scalar RMW.
+; Native <1 x T> elementwise atomicrmw is preserved for backend legalization.
; Shared-memory f32 atom.add preserves denormals, matching the default mode.
define <1 x float> @fadd_v1f32_elementwise(ptr addrspace(3) %addr, <1 x float> %val) {
; CHECK-LABEL: @fadd_v1f32_elementwise(
@@ -16,7 +16,7 @@ entry:
ret <1 x float> %old
}
-; A <2 x float> elementwise fadd in shared memory splits into native scalar atomics.
+; Shared-memory fadd splits into one-lane atomics for backend legalization.
define <2 x float> @fadd_v2f32_elementwise(ptr addrspace(3) %addr, <2 x float> %val) {
; CHECK-LABEL: @fadd_v2f32_elementwise(
; CHECK-NEXT: entry:
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
index d0c6a98025caa1..466780e10be460 100644
--- a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
@@ -10,9 +10,7 @@
define <4 x i32> @atomicrmw_add_elementwise(ptr %p, <4 x i32> %v) {
; CHECK-LABEL: define <4 x i32> @atomicrmw_add_elementwise(
; CHECK-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[V]] to i128
-; CHECK-NEXT: [[TMP2:%.*]] = call i128 @__atomic_fetch_add_16(ptr [[P]], i128 [[TMP1]], i32 0)
-; CHECK-NEXT: [[TMP3:%.*]] = bitcast i128 [[TMP2]] to <4 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = atomicrmw elementwise add ptr [[P]], <4 x i32> [[V]] monotonic, align 16
; CHECK-NEXT: ret <4 x i32> [[TMP3]]
;
%old = atomicrmw elementwise add ptr %p, <4 x i32> %v monotonic
@@ -22,24 +20,7 @@ define <4 x i32> @atomicrmw_add_elementwise(ptr %p, <4 x i32> %v) {
define <4 x float> @atomicrmw_fadd_elementwise(ptr %p, <4 x float> %v) {
; CHECK-LABEL: define <4 x float> @atomicrmw_fadd_elementwise(
; CHECK-SAME: ptr [[P:%.*]], <4 x float> [[V:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = alloca <4 x float>, align 16
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[P]], align 16
-; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
-; CHECK: [[ATOMICRMW_START]]:
-; CHECK-NEXT: [[LOADED:%.*]] = phi <4 x float> [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
-; CHECK-NEXT: [[NEW:%.*]] = fadd <4 x float> [[LOADED]], [[V]]
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP1]])
-; CHECK-NEXT: store <4 x float> [[LOADED]], ptr [[TMP1]], align 16
-; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x float> [[NEW]] to i128
-; CHECK-NEXT: [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_16(ptr [[P]], ptr [[TMP1]], i128 [[TMP3]], i32 0, i32 0)
-; CHECK-NEXT: [[TMP5:%.*]] = load <4 x float>, ptr [[TMP1]], align 16
-; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP1]])
-; CHECK-NEXT: [[TMP6:%.*]] = insertvalue { <4 x float>, i1 } poison, <4 x float> [[TMP5]], 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertvalue { <4 x float>, i1 } [[TMP6]], i1 [[TMP4]], 1
-; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { <4 x float>, i1 } [[TMP7]], 1
-; CHECK-NEXT: [[NEWLOADED]] = extractvalue { <4 x float>, i1 } [[TMP7]], 0
-; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
-; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: [[NEWLOADED:%.*]] = atomicrmw elementwise fadd ptr [[P]], <4 x float> [[V]] monotonic, align 16
; CHECK-NEXT: ret <4 x float> [[NEWLOADED]]
;
%old = atomicrmw elementwise fadd ptr %p, <4 x float> %v monotonic
More information about the llvm-branch-commits
mailing list