[llvm-branch-commits] [llvm] [AtomicExpand][NVPTX] Support native elementwise atomicrmw (PR #233742)
Yonah Goldberg via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Oct 20 01:30:30 PDT 2026
https://github.com/YonahGoldberg created https://github.com/llvm/llvm-project/pull/233742
None
>From 264d1ddb30b755b9beac304f8e094a0ffd070086 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 20 Oct 2026 08:24:36 +0000
Subject: [PATCH] [AtomicExpand][NVPTX] Support native elementwise atomicrmw
---
llvm/include/llvm/CodeGen/TargetLowering.h | 4 +
llvm/lib/CodeGen/AtomicExpandPass.cpp | 115 +-
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 152 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 309 +-
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 47 +-
.../Target/NVPTX/NVPTXSelectionDAGInfo.cpp | 3 +
llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h | 9 +-
.../NVPTX/atomicrmw-elementwise-sm60.ll | 4556 ++++++++--------
.../NVPTX/atomicrmw-elementwise-sm70.ll | 4830 ++++++++---------
.../NVPTX/atomicrmw-elementwise-sm90.ll | 3399 ++++--------
llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll | 95 +
.../NVPTX/expand-atomic-rmw-elementwise.ll | 79 +-
12 files changed, 6287 insertions(+), 7311 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 2efa229cdc930e..28f45d7043e8af 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -2288,6 +2288,8 @@ class LLVM_ABI TargetLoweringBase {
/// the backend. Atomic operations greater than this size (as well
/// as ones that are not naturally aligned), will be expanded by
/// AtomicExpandPass into an __atomic_* library call.
+ /// For elementwise RMWs, the limit applies to each lane. Whole-vector
+ /// cmpxchg/LL-SC expansions must also satisfy it for their full access size.
unsigned getMaxAtomicSizeInBitsSupported() const {
return MaxAtomicSizeInBitsSupported;
}
@@ -2549,6 +2551,8 @@ class LLVM_ABI TargetLoweringBase {
/// Returns how the IR-level AtomicExpand pass should expand the given
/// AtomicRMW, if at all. By default, elementwise operations are expanded and
/// floating-point operations use are expanded to a compare-exchange loop.
+ /// For native elementwise RMWs (None), the target must check the vector type,
+ /// width and alignment; generic size checks only validate the lane size.
virtual AtomicExpansionKind
shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
if (RMW->isElementwise())
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index d4a9f0df80ab0c..01d9e0845d855c 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -214,25 +214,24 @@ INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
INITIALIZE_PASS_END(AtomicExpandLegacy, DEBUG_TYPE,
"Expand Atomic instructions", false, false)
-// Helper functions to retrieve the size of atomic instructions.
-static unsigned getAtomicOpSize(LoadInst *LI) {
- const DataLayout &DL = LI->getDataLayout();
- return DL.getTypeStoreSize(LI->getType());
+// Helper functions to retrieve the type accessed by atomic instructions.
+static Type *getAtomicOpType(LoadInst *LI) { return LI->getType(); }
+
+static Type *getAtomicOpType(StoreInst *SI) {
+ return SI->getValueOperand()->getType();
}
-static unsigned getAtomicOpSize(StoreInst *SI) {
- const DataLayout &DL = SI->getDataLayout();
- return DL.getTypeStoreSize(SI->getValueOperand()->getType());
+static Type *getAtomicOpType(AtomicRMWInst *RMWI) {
+ return RMWI->getValOperand()->getType();
}
-static unsigned getAtomicOpSize(AtomicRMWInst *RMWI) {
- const DataLayout &DL = RMWI->getDataLayout();
- return DL.getTypeStoreSize(RMWI->getValOperand()->getType());
+static Type *getAtomicOpType(AtomicCmpXchgInst *CASI) {
+ return CASI->getCompareOperand()->getType();
}
-static unsigned getAtomicOpSize(AtomicCmpXchgInst *CASI) {
- const DataLayout &DL = CASI->getDataLayout();
- return DL.getTypeStoreSize(CASI->getCompareOperand()->getType());
+// The storage size of the full access, including all lanes of a vector.
+template <typename Inst> static unsigned getTypeStoreSize(Inst *I) {
+ return I->getDataLayout().getTypeStoreSize(getAtomicOpType(I));
}
/// Copy metadata that's safe to preserve when widening atomics.
@@ -254,6 +253,7 @@ static void copyMetadataForAtomic(Instruction &Dest,
case LLVMContext::MD_noalias_addrspace:
case LLVMContext::MD_access_group:
case LLVMContext::MD_mmra:
+ case LLVMContext::MD_atomic_ignore_denormal_mode:
Dest.setMetadata(ID, N);
break;
default:
@@ -262,25 +262,32 @@ static void copyMetadataForAtomic(Instruction &Dest,
else if (ID == Ctx.getMDKindID("amdgpu.no.fine.grained.memory"))
Dest.setMetadata(ID, N);
- // Losing atomic.ignore.denormal.mode, but it doesn't matter for current
- // uses.
break;
}
}
}
-template <typename Inst>
-static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
- unsigned Size = getAtomicOpSize(I);
- Align Alignment = I->getAlign();
+static bool atomicSizeSupported(const TargetLowering *TLI, unsigned Size,
+ Align Alignment) {
unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8;
return Alignment >= Size && Size <= MaxSize;
}
+template <typename Inst>
+static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
+ Type *Ty = getAtomicOpType(I);
+ // Elementwise RMWs require atomic support for each lane. Expansions that
+ // access the whole vector atomically validate that access separately.
+ if (auto *RMWI = dyn_cast<AtomicRMWInst>(I); RMWI && RMWI->isElementwise())
+ Ty = Ty->getScalarType();
+ unsigned Size = I->getDataLayout().getTypeStoreSize(Ty);
+ return atomicSizeSupported(TLI, Size, I->getAlign());
+}
+
template <typename Inst>
static void writeUnsupportedAtomicSizeReason(const TargetLowering *TLI, Inst *I,
raw_ostream &OS) {
- unsigned Size = getAtomicOpSize(I);
+ unsigned Size = getTypeStoreSize(I);
Align Alignment = I->getAlign();
bool NeedSeparator = false;
@@ -936,6 +943,10 @@ void AtomicExpandImpl::expandAtomicSubToAdd(AtomicRMWInst *AI) {
AI->setOperation(NewOp);
AI->setOperand(1, NewVal);
+
+ // Re-query the target for the addition, which may be native or require
+ // further expansion, including splitting an elementwise vector.
+ processAtomicInstr(AI);
}
bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
@@ -946,7 +957,11 @@ bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
return false;
case TargetLoweringBase::AtomicExpansionKind::LLSC: {
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
- unsigned ValueSize = getAtomicOpSize(AI);
+ unsigned ValueSize = getTypeStoreSize(AI);
+ if (!atomicSizeSupported(TLI, ValueSize, AI->getAlign())) {
+ expandAtomicRMWToLibcall(AI);
+ return true;
+ }
if (ValueSize < MinCASSize) {
expandPartwordAtomicRMW(AI,
TargetLoweringBase::AtomicExpansionKind::LLSC);
@@ -962,7 +977,11 @@ bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
}
case TargetLoweringBase::AtomicExpansionKind::CmpXChg: {
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
- unsigned ValueSize = getAtomicOpSize(AI);
+ unsigned ValueSize = getTypeStoreSize(AI);
+ if (!atomicSizeSupported(TLI, ValueSize, AI->getAlign())) {
+ expandAtomicRMWToLibcall(AI);
+ return true;
+ }
if (ValueSize < MinCASSize) {
expandPartwordAtomicRMW(AI,
TargetLoweringBase::AtomicExpansionKind::CmpXChg);
@@ -985,7 +1004,7 @@ bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
}
case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic: {
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
- unsigned ValueSize = getAtomicOpSize(AI);
+ unsigned ValueSize = getTypeStoreSize(AI);
if (ValueSize < MinCASSize) {
AtomicRMWInst::BinOp Op = AI->getOperation();
// Widen And/Or/Xor and give the target another chance at expanding it.
@@ -1007,11 +1026,13 @@ bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
return true;
}
case TargetLoweringBase::AtomicExpansionKind::Expand:
- if (AI->isElementwise()) {
- expandElementwiseAtomicRMW(AI);
+ if (AI->getOperation() == AtomicRMWInst::Sub ||
+ AI->getOperation() == AtomicRMWInst::FSub) {
+ expandAtomicSubToAdd(AI);
return true;
}
- expandAtomicSubToAdd(AI);
+ assert(AI->isElementwise() && "Expected an elementwise atomicrmw");
+ expandElementwiseAtomicRMW(AI);
return true;
case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
return lowerAtomicRMWInst(AI);
@@ -1178,6 +1199,20 @@ static Value *insertMaskedValue(IRBuilderBase &Builder, Value *WideWord,
return Or;
}
+/// Whether a masked operation can use a shifted operand without extracting the
+/// subword value on each iteration.
+static bool canUseShiftedOperand(AtomicRMWInst::BinOp Op, Type *ValueType) {
+ if (Op == AtomicRMWInst::Xchg || Op == AtomicRMWInst::Nand)
+ return true;
+ if (Op != AtomicRMWInst::Add && Op != AtomicRMWInst::Sub)
+ return false;
+
+ // Whole-word arithmetic is safe with one lane. With multiple lanes, carries
+ // or borrows could cross lane boundaries.
+ auto *VecTy = dyn_cast<VectorType>(ValueType);
+ return !VecTy || VecTy->getElementCount() == ElementCount::getFixed(1);
+}
+
/// Emit IR to implement a masked version of a given atomicrmw
/// operation. (That is, only the bits under the Mask should be
/// affected by the operation)
@@ -1203,18 +1238,15 @@ static Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op,
return FinalVal;
}
- if (Op == AtomicRMWInst::Nand ||
- (!PMV.ValueType->isVectorTy() &&
- (Op == AtomicRMWInst::Add || Op == AtomicRMWInst::Sub))) {
- // For `Nand` and non-vector `Add` and `Sub`, we can perform the operation
+ if (canUseShiftedOperand(Op, PMV.ValueType)) {
+ // For `Nand` and single-lane `Add` and `Sub`, we can perform the operation
// on the entire word because the extra bits in the unmasked region don't
// affect the computation in the masked region. The operation might still
// overwrite the unmasked region (e.g. from integer overflow or underflow),
// so we have to reapply the unmasked region afterwards.
//
- // This trick doesn't work for vector `Add` and `Sub` because we use a
- // scalar operation on the entire word. Scalarizing vector `Add` and `Sub`
- // isn't legal because the vector versions may have element-wise overflows.
+ // This trick doesn't work for multi-lane vector `Add` and `Sub` because
+ // scalar arithmetic on the entire word could carry or borrow across lanes.
// TODO: For these, can we use a wider vector op with additional lanes?
// Atomic operation across the entire word.
@@ -1263,12 +1295,7 @@ void AtomicExpandImpl::expandPartwordAtomicRMW(
AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
Value *ValOperand_Shifted = nullptr;
- bool NeedsShiftedOperand =
- Op == AtomicRMWInst::Xchg || Op == AtomicRMWInst::Nand ||
- (!PMV.ValueType->isVectorTy() &&
- (Op == AtomicRMWInst::Add || Op == AtomicRMWInst::Sub));
-
- if (NeedsShiftedOperand) {
+ if (canUseShiftedOperand(Op, PMV.ValueType)) {
Value *ValOp = Builder.CreateBitCast(AI->getValOperand(), PMV.IntValueType);
ValOperand_Shifted =
Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
@@ -2009,7 +2036,7 @@ Value *AtomicExpandImpl::insertRMWCmpXchgLoop(
bool AtomicExpandImpl::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
- unsigned ValueSize = getAtomicOpSize(CI);
+ unsigned ValueSize = getTypeStoreSize(CI);
switch (TLI->shouldExpandAtomicCmpXchgInIR(CI)) {
default:
@@ -2079,7 +2106,7 @@ void AtomicExpandImpl::expandAtomicLoadToLibcall(LoadInst *I) {
static const RTLIB::Libcall Libcalls[6] = {
RTLIB::ATOMIC_LOAD, RTLIB::ATOMIC_LOAD_1, RTLIB::ATOMIC_LOAD_2,
RTLIB::ATOMIC_LOAD_4, RTLIB::ATOMIC_LOAD_8, RTLIB::ATOMIC_LOAD_16};
- unsigned Size = getAtomicOpSize(I);
+ unsigned Size = getTypeStoreSize(I);
bool Expanded = expandAtomicOpToLibcall(
I, Size, I->getAlign(), I->getPointerOperand(), nullptr, nullptr,
@@ -2092,7 +2119,7 @@ void AtomicExpandImpl::expandAtomicStoreToLibcall(StoreInst *I) {
static const RTLIB::Libcall Libcalls[6] = {
RTLIB::ATOMIC_STORE, RTLIB::ATOMIC_STORE_1, RTLIB::ATOMIC_STORE_2,
RTLIB::ATOMIC_STORE_4, RTLIB::ATOMIC_STORE_8, RTLIB::ATOMIC_STORE_16};
- unsigned Size = getAtomicOpSize(I);
+ unsigned Size = getTypeStoreSize(I);
bool Expanded = expandAtomicOpToLibcall(
I, Size, I->getAlign(), I->getPointerOperand(), I->getValueOperand(),
@@ -2108,7 +2135,7 @@ void AtomicExpandImpl::expandAtomicCASToLibcall(AtomicCmpXchgInst *I,
RTLIB::ATOMIC_COMPARE_EXCHANGE, RTLIB::ATOMIC_COMPARE_EXCHANGE_1,
RTLIB::ATOMIC_COMPARE_EXCHANGE_2, RTLIB::ATOMIC_COMPARE_EXCHANGE_4,
RTLIB::ATOMIC_COMPARE_EXCHANGE_8, RTLIB::ATOMIC_COMPARE_EXCHANGE_16};
- unsigned Size = getAtomicOpSize(I);
+ unsigned Size = getTypeStoreSize(I);
bool Expanded = expandAtomicOpToLibcall(
I, Size, I->getAlign(), I->getPointerOperand(), I->getNewValOperand(),
@@ -2190,7 +2217,7 @@ static ArrayRef<RTLIB::Libcall> GetRMWLibcall(AtomicRMWInst::BinOp Op) {
void AtomicExpandImpl::expandAtomicRMWToLibcall(AtomicRMWInst *I) {
ArrayRef<RTLIB::Libcall> Libcalls = GetRMWLibcall(I->getOperation());
- unsigned Size = getAtomicOpSize(I);
+ unsigned Size = getTypeStoreSize(I);
bool Success = false;
if (!Libcalls.empty())
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 848f819c5503e9..61bcc36225ec7f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -78,7 +78,13 @@ struct NVPTXMemCacheHintAccess {
NVPTX::AddressSpace AddrSpace;
unsigned NumElts;
unsigned EltWidth;
- bool IsVolatile;
+ // Ordering of the selected PTX instruction, which may drop IR volatility.
+ NVPTX::Ordering Ordering;
+
+ bool isVolatileOrMMIO() const {
+ return Ordering == NVPTX::Ordering::Volatile ||
+ Ordering == NVPTX::Ordering::RelaxedMMIO;
+ }
};
struct NVPTXMemCacheHintOperands {
@@ -135,6 +141,7 @@ class NVPTXDAGToDAGISel : public SelectionDAGISel {
void SelectTcgen05Ld(SDNode *N, bool hasOffset = false);
void SelectTcgen05St(SDNode *N, bool hasOffset = false);
void selectAtomicSwap128(SDNode *N);
+ void selectVectorAtomicRMW(SDNode *N);
inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) {
return CurDAG->getTargetConstant(Imm, DL, MVT::i32);
@@ -318,6 +325,11 @@ void NVPTXDAGToDAGISel::Select(SDNode *N) {
case NVPTXISD::ATOMIC_SWAP_B128:
selectAtomicSwap128(N);
return;
+ case NVPTXISD::ATOM_VEC_V1:
+ case NVPTXISD::ATOM_VEC_V2:
+ case NVPTXISD::ATOM_VEC_V4:
+ selectVectorAtomicRMW(N);
+ return;
case ISD::FADD:
case ISD::FMUL:
case ISD::FSUB:
@@ -1233,13 +1245,17 @@ static bool isL1EvictionSupported(const NVPTXSubtarget &Subtarget,
if (Eviction == NVPTX::L1Eviction::Normal)
return true;
- return isLdOrSt(Access) && !Access.IsVolatile &&
+ return isLdOrSt(Access) && !Access.isVolatileOrMMIO() &&
Subtarget.hasL1EvictionHint();
}
static bool isL2PrefetchSupported(const NVPTXSubtarget &Subtarget,
NVPTX::L2Prefetch Prefetch,
NVPTXMemCacheHintAccess Access) {
+ if (Prefetch != NVPTX::L2Prefetch::None &&
+ Access.Ordering == NVPTX::Ordering::RelaxedMMIO)
+ return false;
+
switch (Prefetch) {
case NVPTX::L2Prefetch::None:
return true;
@@ -1262,7 +1278,7 @@ static bool isL2EvictionSupported(const NVPTXSubtarget &Subtarget,
if (Eviction == NVPTX::L2Eviction::Normal)
return true;
- return isLdOrSt(Access) && !Access.IsVolatile &&
+ return isLdOrSt(Access) && !Access.isVolatileOrMMIO() &&
Subtarget.hasL2EvictionHint() && isGlobalOrGeneric(Access.AddrSpace) &&
((Access.NumElts == 8 && Access.EltWidth == 32) ||
(Access.NumElts == 4 && Access.EltWidth == 64));
@@ -1270,7 +1286,7 @@ static bool isL2EvictionSupported(const NVPTXSubtarget &Subtarget,
static bool isCachePolicySupported(const NVPTXSubtarget &Subtarget,
NVPTXMemCacheHintAccess Access) {
- return !Access.IsVolatile && isGlobalOrGeneric(Access.AddrSpace) &&
+ return !Access.isVolatileOrMMIO() && isGlobalOrGeneric(Access.AddrSpace) &&
Subtarget.hasL2CacheHint();
}
@@ -1397,7 +1413,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
LD,
{NVPTXMemCacheHintInstruction::Ld, CodeAddrSpace,
- /*NumElts=*/1, /*EltWidth=*/FromTypeWidth, LD->isVolatile()},
+ /*NumElts=*/1, /*EltWidth=*/FromTypeWidth, Ordering},
DL);
// Create the machine instruction DAG
@@ -1478,7 +1494,7 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
getMemCacheHintOperands(LD,
{NVPTXMemCacheHintInstruction::Ld, CodeAddrSpace,
/*NumElts=*/LD->getNumValues() - 1,
- /*EltWidth=*/FromTypeWidth, LD->isVolatile()},
+ /*EltWidth=*/FromTypeWidth, Ordering},
DL);
const auto [Base, Offset] = selectADDR(N->getOperand(1), CurDAG);
SDValue Ops[] = {getI32Imm(Ordering, DL),
@@ -1547,7 +1563,7 @@ bool NVPTXDAGToDAGISel::tryLDG(MemSDNode *LD) {
const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
LD,
{NVPTXMemCacheHintInstruction::Ld, NVPTX::AddressSpace::Global,
- LD->getNumValues() - 1, FromTypeWidth, LD->isVolatile()},
+ LD->getNumValues() - 1, FromTypeWidth, NVPTX::Ordering::NotAtomic},
DL);
SDValue Ops[] = {getI32Imm(FromType, DL),
getI32Imm(FromTypeWidth, DL),
@@ -1669,7 +1685,7 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
ST,
{NVPTXMemCacheHintInstruction::St, CodeAddrSpace,
- /*NumElts=*/1, /*EltWidth=*/ToTypeWidth, ST->isVolatile()},
+ /*NumElts=*/1, /*EltWidth=*/ToTypeWidth, Ordering},
DL);
SDValue Ops[] = {selectPossiblyImm(Value),
@@ -1730,7 +1746,7 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
ST,
{NVPTXMemCacheHintInstruction::St, CodeAddrSpace,
- /*NumElts=*/NumElts, /*EltWidth=*/ToTypeWidth, ST->isVolatile()},
+ /*NumElts=*/NumElts, /*EltWidth=*/ToTypeWidth, Ordering},
DL);
const auto [Base, Offset] = selectADDR(Addr, CurDAG);
@@ -2352,6 +2368,122 @@ bool NVPTXDAGToDAGISel::tryIntrinsicVoid(SDNode *N) {
}
}
+void NVPTXDAGToDAGISel::selectVectorAtomicRMW(SDNode *N) {
+ MemSDNode *MN = cast<MemSDNode>(N);
+ SDLoc DL(N);
+
+ unsigned NumRegs;
+ switch (N->getOpcode()) {
+ case NVPTXISD::ATOM_VEC_V1:
+ NumRegs = 1;
+ break;
+ case NVPTXISD::ATOM_VEC_V2:
+ NumRegs = 2;
+ break;
+ case NVPTXISD::ATOM_VEC_V4:
+ NumRegs = 4;
+ break;
+ default:
+ llvm_unreachable("Unexpected opcode");
+ }
+
+ // Node layout: chain, ptr, val0..valN-1, op_code
+ const auto [Base, Offset] = selectADDR(N->getOperand(1), CurDAG);
+ unsigned OpCode =
+ cast<ConstantSDNode>(N->getOperand(2 + NumRegs))->getZExtValue();
+ MVT RegVT = N->getSimpleValueType(0);
+
+ // Opcode table indexed by [op][register_type][register_count].
+ // op: 0=fadd, 1=fmin, 2=fmax
+ // register_type: 0=f32, 1=f16, 2=bf16, 3=f16x2, 4=bf16x2
+ // register_count: 0=1, 1=2, 2=4
+ // clang-format off
+ static const unsigned OpcTable[3][5][3] = {
+ // fadd
+ {{0, NVPTX::ATOM_ADD_V2_F32, NVPTX::ATOM_ADD_V4_F32},
+ {0, 0, 0},
+ {0, 0, 0},
+ {NVPTX::INT_PTX_ATOM_ADD_F16X2_r, NVPTX::ATOM_ADD_V2_F16X2, NVPTX::ATOM_ADD_V4_F16X2},
+ {NVPTX::INT_PTX_ATOM_ADD_BF16X2_r, NVPTX::ATOM_ADD_V2_BF16X2, NVPTX::ATOM_ADD_V4_BF16X2}},
+ // fmin
+ {{0, 0, 0},
+ {0, NVPTX::ATOM_MIN_V2_F16, 0},
+ {0, NVPTX::ATOM_MIN_V2_BF16, 0},
+ {0, NVPTX::ATOM_MIN_V2_F16X2, NVPTX::ATOM_MIN_V4_F16X2},
+ {0, NVPTX::ATOM_MIN_V2_BF16X2, NVPTX::ATOM_MIN_V4_BF16X2}},
+ // fmax
+ {{0, 0, 0},
+ {0, NVPTX::ATOM_MAX_V2_F16, 0},
+ {0, NVPTX::ATOM_MAX_V2_BF16, 0},
+ {0, NVPTX::ATOM_MAX_V2_F16X2, NVPTX::ATOM_MAX_V4_F16X2},
+ {0, NVPTX::ATOM_MAX_V2_BF16X2, NVPTX::ATOM_MAX_V4_BF16X2}},
+ };
+ // clang-format on
+
+ unsigned TypeIdx;
+ switch (RegVT.SimpleTy) {
+ case MVT::f32:
+ TypeIdx = 0;
+ break;
+ case MVT::f16:
+ TypeIdx = 1;
+ break;
+ case MVT::bf16:
+ TypeIdx = 2;
+ break;
+ case MVT::v2f16:
+ TypeIdx = 3;
+ break;
+ case MVT::v2bf16:
+ TypeIdx = 4;
+ break;
+ default:
+ report_fatal_error("Unsupported register type for vector atom");
+ }
+
+ unsigned WidthIdx;
+ switch (NumRegs) {
+ case 1:
+ WidthIdx = 0;
+ break;
+ case 2:
+ WidthIdx = 1;
+ break;
+ case 4:
+ WidthIdx = 2;
+ break;
+ default:
+ report_fatal_error("Unsupported vector width for vector atom");
+ }
+
+ assert(OpCode < std::size(OpcTable) && "Unexpected atomic operation");
+ unsigned Opc = OpcTable[OpCode][TypeIdx][WidthIdx];
+ if (!Opc)
+ report_fatal_error("Unsupported vector atom combination");
+
+ if (RegVT == MVT::f32 && Subtarget->hasFeature(NVPTX::PTX94) && !useF32FTZ())
+ Opc = NumRegs == 2 ? NVPTX::ATOM_ADD_NOFTZ_V2_F32
+ : NVPTX::ATOM_ADD_NOFTZ_V4_F32;
+
+ SmallVector<SDValue, 12> Ops{Base, Offset};
+ for (unsigned I = 0; I < NumRegs; ++I)
+ Ops.push_back(N->getOperand(2 + I));
+ Ops.push_back(getI32Imm(getMemOrder(MN), DL));
+ Ops.push_back(getI32Imm(getAtomicScope(MN), DL));
+ Ops.push_back(getI32Imm(getAddrSpace(MN), DL));
+ NVPTXMemCacheHintAccess Access{
+ NVPTXMemCacheHintInstruction::Atom, getAddrSpace(MN), NumRegs,
+ unsigned(RegVT.getFixedSizeInBits()), getMemOrder(MN)};
+ const auto [EvictionAndPrefetchHint, CachePolicyReg] =
+ getMemCacheHintOperands(MN, Access, DL);
+ Ops.append(
+ {EvictionAndPrefetchHint, CachePolicyReg, N->getOperand(0)}); // chain
+
+ auto *ATOM = CurDAG->getMachineNode(Opc, DL, N->getVTList(), Ops);
+ CurDAG->setNodeMemRefs(ATOM, MN->getMemOperand());
+ ReplaceNode(N, ATOM);
+}
+
void NVPTXDAGToDAGISel::selectAtomicSwap128(SDNode *N) {
MemSDNode *AN = cast<MemSDNode>(N);
SDLoc dl(N);
@@ -2367,7 +2499,7 @@ void NVPTXDAGToDAGISel::selectAtomicSwap128(SDNode *N) {
unsigned EltWidth = AN->getMemoryVT().getFixedSizeInBits();
NVPTXMemCacheHintAccess Access{NVPTXMemCacheHintInstruction::Atom,
getAddrSpace(AN),
- /*NumElts=*/1, EltWidth, AN->isVolatile()};
+ /*NumElts=*/1, EltWidth, getMemOrder(AN)};
const auto [EvictionAndPrefetchHint, CachePolicyReg] =
getMemCacheHintOperands(AN, Access, dl);
Ops.push_back(EvictionAndPrefetchHint);
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 58340d62bb6fea..c738a40b0e118d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -1148,6 +1148,13 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
setOperationAction({ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP}, MVT::i128,
Custom);
+ setOperationAction({ISD::ATOMIC_LOAD_FADD, ISD::ATOMIC_LOAD_FMIN,
+ ISD::ATOMIC_LOAD_FMAX, ISD::ATOMIC_LOAD_FMINIMUMNUM,
+ ISD::ATOMIC_LOAD_FMAXIMUMNUM},
+ {MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v2bf16,
+ MVT::v4bf16, MVT::v8bf16, MVT::v2f32, MVT::v4f32},
+ Custom);
+
// Now deduce the information based on the above mentioned
// actions
computeRegisterProperties(STI.getRegisterInfo());
@@ -3641,9 +3648,106 @@ static SDValue lowerMSTORE(SDValue Op, SelectionDAG &DAG) {
return NewSt;
}
-SDValue
-NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
+enum AtomVecOp : unsigned {
+ ATOM_VEC_FADD = 0,
+ ATOM_VEC_FMIN = 1,
+ ATOM_VEC_FMAX = 2,
+};
+
+// Replace a vector atomic with a target memory node that returns one value
+// per PTX register, followed by the chain. Half and bfloat values use packed
+// pairs when the instruction supports them.
+static void replaceVectorAtomicRMW(SDNode *N, SelectionDAG &DAG,
+ SmallVectorImpl<SDValue> &Results) {
+ auto *AN = cast<AtomicSDNode>(N);
+ EVT VecVT = AN->getValueType(0);
+ EVT EltVT = VecVT.getVectorElementType();
+ unsigned NumElts = VecVT.getVectorNumElements();
+ SDLoc DL(N);
+
+ bool IsAdd = N->getOpcode() == ISD::ATOMIC_LOAD_FADD;
+ // Packed scalar atomics only support add. Two-element min/max use v2.f16
+ // or v2.bf16; wider vectors use packed pairs for all supported operations.
+ bool Pack = EltVT != MVT::f32 && (IsAdd || NumElts > 2);
+ EVT RegVT = Pack ? EVT::getVectorVT(*DAG.getContext(), EltVT, 2) : EltVT;
+ unsigned NumRegs = Pack ? NumElts / 2 : NumElts;
+
+ unsigned NVPTXOpc;
+ switch (NumRegs) {
+ case 1:
+ NVPTXOpc = NVPTXISD::ATOM_VEC_V1;
+ break;
+ case 2:
+ NVPTXOpc = NVPTXISD::ATOM_VEC_V2;
+ break;
+ case 4:
+ NVPTXOpc = NVPTXISD::ATOM_VEC_V4;
+ break;
+ default:
+ llvm_unreachable("Unsupported vector width for atom");
+ }
+
+ unsigned OpCode;
+ switch (N->getOpcode()) {
+ case ISD::ATOMIC_LOAD_FADD:
+ OpCode = ATOM_VEC_FADD;
+ break;
+ case ISD::ATOMIC_LOAD_FMIN:
+ case ISD::ATOMIC_LOAD_FMINIMUMNUM:
+ OpCode = ATOM_VEC_FMIN;
+ break;
+ case ISD::ATOMIC_LOAD_FMAX:
+ case ISD::ATOMIC_LOAD_FMAXIMUMNUM:
+ OpCode = ATOM_VEC_FMAX;
+ break;
+ default:
+ llvm_unreachable("Unsupported atomic op for vector atom");
+ }
+
+ SmallVector<SDValue, 10> Ops;
+ Ops.push_back(AN->getChain());
+ Ops.push_back(AN->getBasePtr());
+ SDValue Val = AN->getVal();
+ for (unsigned I = 0; I < NumRegs; ++I) {
+ if (RegVT == VecVT)
+ Ops.push_back(Val);
+ else
+ Ops.push_back(DAG.getNode(
+ Pack ? ISD::EXTRACT_SUBVECTOR : ISD::EXTRACT_VECTOR_ELT, DL, RegVT,
+ Val, DAG.getVectorIdxConstant(Pack ? 2 * I : I, DL)));
+ }
+ Ops.push_back(DAG.getTargetConstant(OpCode, DL, MVT::i32));
+
+ SmallVector<EVT, 5> ResVTs(NumRegs, RegVT);
+ ResVTs.push_back(MVT::Other);
+
+ SDValue Result = DAG.getMemIntrinsicNode(NVPTXOpc, DL, DAG.getVTList(ResVTs),
+ Ops, VecVT, AN->getMemOperand());
+
+ SmallVector<SDValue, 8> Elts;
+ for (unsigned I = 0; I < NumRegs; ++I)
+ Elts.push_back(Result.getValue(I));
+
+ SDValue Vec = RegVT == VecVT ? Result
+ : DAG.getNode(Pack ? ISD::CONCAT_VECTORS
+ : ISD::BUILD_VECTOR,
+ DL, VecVT, Elts);
+ Results.push_back(Vec);
+ Results.push_back(Result.getValue(NumRegs));
+}
+
+SDValue NVPTXTargetLowering::LowerOperation(SDValue Op,
+ SelectionDAG &DAG) const {
switch (Op.getOpcode()) {
+ case ISD::ATOMIC_LOAD_FADD:
+ case ISD::ATOMIC_LOAD_FMIN:
+ case ISD::ATOMIC_LOAD_FMAX:
+ case ISD::ATOMIC_LOAD_FMINIMUMNUM:
+ case ISD::ATOMIC_LOAD_FMAXIMUMNUM: {
+ SmallVector<SDValue, 2> Results;
+ replaceVectorAtomicRMW(Op.getNode(), DAG, Results);
+ return DAG.getMergeValues(Results, SDLoc(Op));
+ }
case ISD::RETURNADDR:
return SDValue();
case ISD::FRAMEADDR:
@@ -7934,13 +8038,22 @@ void NVPTXTargetLowering::ReplaceNodeResults(
case ISD::ATOMIC_SWAP:
replaceAtomicSwap128(N, DAG, STI, Results);
return;
+ case ISD::ATOMIC_LOAD_FADD:
+ case ISD::ATOMIC_LOAD_FMIN:
+ case ISD::ATOMIC_LOAD_FMAX:
+ case ISD::ATOMIC_LOAD_FMINIMUMNUM:
+ case ISD::ATOMIC_LOAD_FMAXIMUMNUM: {
+ replaceVectorAtomicRMW(N, DAG, Results);
+ return;
+ }
}
}
/// Returns how NVPTX should expand a scalar atomicrmw.
-static TargetLoweringBase::AtomicExpansionKind getScalarAtomicRMWExpansion(
- AtomicRMWInst::BinOp Op, Type *Ty, unsigned AddressSpace, const Function &F,
- bool IgnoreDenormalMode, const NVPTXSubtarget &STI) {
+static TargetLoweringBase::AtomicExpansionKind
+getScalarAtomicRMWExpansion(AtomicRMWInst::BinOp Op, Type *Ty,
+ unsigned AddressSpace, const Function &F,
+ bool IgnoreFTZMismatch, const NVPTXSubtarget &STI) {
using AtomicExpansionKind = TargetLoweringBase::AtomicExpansionKind;
assert(!Ty->isVectorTy() && "Expected a scalar atomicrmw type");
@@ -7960,12 +8073,6 @@ static TargetLoweringBase::AtomicExpansionKind getScalarAtomicRMWExpansion(
? AtomicExpansionKind::Expand
: AtomicExpansionKind::None;
- // Both the -nvptx-allow-ftz-atomics option and per-instruction
- // !atomic.ignore.denormal.mode say that denormal handling is insignificant
- // here, so atom.add may be used even when its FTZ behavior disagrees with
- // the function's.
- const bool IgnoreFTZMismatch = AllowFTZAtomics || IgnoreDenormalMode;
-
if (Ty->isFloatTy()) {
const bool FTZ = F.getDenormalMode(APFloat::IEEEsingle()).Output ==
DenormalMode::PreserveSign;
@@ -8072,26 +8179,102 @@ static TargetLoweringBase::AtomicExpansionKind getScalarAtomicRMWExpansion(
return AtomicExpansionKind::CmpXChg;
}
+// Return the number of lanes in the widest native elementwise atomic for this
+// lane type/operation. Vector atomics require global memory; a generic pointer
+// can also refer to shared memory, where only the packed f16x2/bf16x2 adds are
+// available.
+static unsigned getWidestNativeAtomicRMWVectorWidth(
+ AtomicRMWInst::BinOp Op, Type *Ty, unsigned AddressSpace, const Function &F,
+ bool IgnoreFTZMismatch, const NVPTXSubtarget &STI) {
+ // PTX vector atom only supports floating-point add/min/max. sub is expanded
+ // to add.
+ bool IsAdd = Op == AtomicRMWInst::FAdd || Op == AtomicRMWInst::FSub;
+ if (!IsAdd && Op != AtomicRMWInst::FMin && Op != AtomicRMWInst::FMax &&
+ Op != AtomicRMWInst::FMinimumNum && Op != AtomicRMWInst::FMaximumNum)
+ return 0;
+
+ bool HasVector = AddressSpace == ADDRESS_SPACE_GLOBAL &&
+ STI.hasFeature(NVPTX::SM90) && STI.hasFeature(NVPTX::PTX81);
+ if (Ty->isFloatTy()) {
+ bool FTZ = F.getDenormalMode(APFloat::IEEEsingle()).Output ==
+ DenormalMode::PreserveSign;
+ return IsAdd && HasVector &&
+ (FTZ || IgnoreFTZMismatch || STI.hasFeature(NVPTX::PTX94))
+ ? 4
+ : 0;
+ }
+
+ if (!Ty->isHalfTy() && !Ty->isBFloatTy())
+ return 0;
+
+ // Native f16/bf16 atomics always preserve denormals. If the function requires
+ // f16 flushing, use a CAS loop unless the mismatch is explicitly allowed.
+ if (Ty->isHalfTy()) {
+ bool FTZ = F.getDenormalMode(APFloat::IEEEhalf()).Output ==
+ DenormalMode::PreserveSign;
+ if (FTZ && !IgnoreFTZMismatch)
+ return 0;
+ }
+
+ // Global vector atomics support up to eight f16/bf16 lanes.
+ if (HasVector)
+ return 8;
+
+ // Otherwise, only packed f16x2/bf16x2 add is available. These forms also
+ // support generic and shared addresses.
+ if (!IsAdd)
+ return 0;
+ if (AddressSpace != ADDRESS_SPACE_GENERIC &&
+ AddressSpace != ADDRESS_SPACE_GLOBAL &&
+ AddressSpace != ADDRESS_SPACE_SHARED)
+ return 0;
+
+ bool HasPackedAdd =
+ Ty->isHalfTy()
+ ? STI.hasFeature(NVPTX::SM60) && STI.hasFeature(NVPTX::PTX62)
+ : STI.hasFeature(NVPTX::SM90) && STI.hasFeature(NVPTX::PTX78);
+ return HasPackedAdd ? 2 : 0;
+}
+
NVPTXTargetLowering::AtomicExpansionKind
NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
const DataLayout &DL = AI->getDataLayout();
AtomicRMWInst::BinOp Op = AI->getOperation();
- const bool IgnoreDenormalMode =
+ // atom.add may be used even when its FTZ behavior disagrees with
+ // the functions'.
+ const bool IgnoreFTZMismatch =
+ AllowFTZAtomics ||
AI->hasMetadata(LLVMContext::MD_atomic_ignore_denormal_mode);
- // TODO: once we support native elementwise vector atoms,
- // return `AtomicExpansionKind::None` to preserve and lower them.
if (AI->isElementwise()) {
auto *VecTy = cast<FixedVectorType>(AI->getType());
Type *Ty = VecTy->getElementType();
- // Collapse <1 x T> elementwise RMWs to scalar RMWs
+ // Single-lane vectors have the scalar operation's expansion requirements.
+ // Native operations are scalarized by SelectionDAG type legalization.
if (VecTy->getNumElements() == 1)
- return AtomicExpansionKind::Expand;
+ return getScalarAtomicRMWExpansion(Op, Ty, AI->getPointerAddressSpace(),
+ *AI->getFunction(), IgnoreFTZMismatch,
+ STI);
uint64_t VecBytes = DL.getTypeStoreSize(VecTy).getFixedValue();
uint64_t VecBits = DL.getTypeStoreSizeInBits(VecTy).getFixedValue();
+ unsigned NativeWidth = getWidestNativeAtomicRMWVectorWidth(
+ Op, Ty, AI->getPointerAddressSpace(), *AI->getFunction(),
+ IgnoreFTZMismatch, STI);
+ if (NativeWidth) {
+ // Rewrite subtraction to addition in IR before preserving or splitting
+ // the native vector atomic.
+ if (Op == AtomicRMWInst::FSub)
+ return AtomicExpansionKind::Expand;
+ if (VecTy->getNumElements() <= NativeWidth && AI->getAlign() >= VecBytes)
+ return AtomicExpansionKind::None;
+ // Halving will find a native vector width or a scalar operation with
+ // sufficient alignment.
+ return AtomicExpansionKind::Expand;
+ }
+
if (Op == AtomicRMWInst::BinOp::Xchg) {
// Split exchanges that are too wide or insufficiently aligned until
// each piece can be lowered atomically.
@@ -8109,20 +8292,21 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
// If the scalar lane op is natively supported or expands to one, return
// Expand so halving eventually bottoms out at the scalar base case. The
// latter includes sub and fsub, which expand to add and fadd, respectively.
- AtomicExpansionKind LaneExpansionKind = getScalarAtomicRMWExpansion(
- Op, Ty, AI->getPointerAddressSpace(), *AI->getFunction(),
- IgnoreDenormalMode, STI);
+ AtomicExpansionKind LaneExpansionKind =
+ getScalarAtomicRMWExpansion(Op, Ty, AI->getPointerAddressSpace(),
+ *AI->getFunction(), IgnoreFTZMismatch, STI);
if (LaneExpansionKind == AtomicExpansionKind::None ||
LaneExpansionKind == AtomicExpansionKind::Expand)
return AtomicExpansionKind::Expand;
- // If the whole vector fits a single native cmpxchg, emit one wide
- // cmpxchg loop at the current width.
- if (VecBits <= getMaxAtomicSizeInBitsSupported())
+ // If the whole vector fits a single native cmpxchg and has sufficient
+ // alignment, emit one wide cmpxchg loop at the current width.
+ if (VecBits <= getMaxAtomicSizeInBitsSupported() &&
+ AI->getAlign() >= VecBytes)
return AtomicExpansionKind::CmpXChg;
- // If the vector is too wide for a single native cmpxchg, halve further to
- // emit multiple cmpxchg loops.
+ // Otherwise, halve until each piece is small enough and sufficiently
+ // aligned for a native cmpxchg.
return AtomicExpansionKind::Expand;
}
@@ -8130,48 +8314,55 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
if (Ty->isVectorTy())
return AtomicExpansionKind::CmpXChg;
return getScalarAtomicRMWExpansion(Op, Ty, AI->getPointerAddressSpace(),
- *AI->getFunction(), IgnoreDenormalMode,
+ *AI->getFunction(), IgnoreFTZMismatch,
STI);
}
bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
const Instruction *I) const {
- // This function returns true iff AtomicExpandPass should enforce the
- // instruction's ordering with fences instead of leaving that ordering on the
- // atomic instruction itself. This covers CAS-loop emulation, seq_cst
- // operations (which are not natively supported by the PTX `atom`
- // instruction), and elementwise atomicrmw expansion.
- //
- // atomicrmw and cmpxchg instructions not efficiently supported by PTX
- // are lowered to CAS emulation loops that preserve their memory order,
- // syncscope, and volatile semantics. For PTX, it is more efficient to use
- // atom.cas.relaxed.sco instructions within the loop, and fences before and
- // after the loop to restore order.
- //
- // For ordered elementwise atomicrmw, if we expand elementwise, it is also
- // more efficient to insert fences around the whole split sequence once and
- // downgrade the ordering for the entire sequence, rather than having the
- // stronger ordering on every expanded atomicrmw.
- //
- // On targets with memory-order qualifiers, atomic instructions efficiently
- // supported by PTX are lowered to `atom.<op>.<sem>.<scope>` instructions with
- // their corresponding memory order and scope. Since PTX does not support
- // seq_cst, we emulate it by lowering to a fence.sc followed by an atom
- // according to the PTX atomics ABI.
- // https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
- if (auto *CI = dyn_cast<AtomicCmpXchgInst>(I))
- return (cast<IntegerType>(CI->getCompareOperand()->getType())
- ->getBitWidth() < STI.getMinCmpXchgSizeInBits()) ||
- !STI.hasMemoryOrdering() ||
- CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent;
- if (auto *RI = dyn_cast<AtomicRMWInst>(I)) {
+ // A true result lets AtomicExpand enforce the original ordering with fences
+ // and weaken the atomic itself. The pass only emits fences for ordered
+ // operations.
+ AtomicOrdering Ordering;
+ if (auto *CI = dyn_cast<AtomicCmpXchgInst>(I)) {
+ // Subword cmpxchg needs a CAS loop. Place fences outside the loop so its
+ // retries can use relaxed atomics.
+ if (cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth() <
+ STI.getMinCmpXchgSizeInBits())
+ return true;
+ Ordering = CI->getMergedOrdering();
+ } else if (auto *RI = dyn_cast<AtomicRMWInst>(I)) {
AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(RI);
- return Expansion == AtomicExpansionKind::CmpXChg ||
- (RI->isElementwise() && Expansion == AtomicExpansionKind::Expand) ||
- !STI.hasMemoryOrdering() ||
- RI->getOrdering() == AtomicOrdering::SequentiallyConsistent;
+ // Enforce ordering around the entire RMW loop, leaving each CAS relaxed.
+ if (Expansion == AtomicExpansionKind::CmpXChg)
+ return true;
+
+ if (RI->isElementwise() && Expansion == AtomicExpansionKind::Expand) {
+ // AtomicExpand reprocesses subtraction after rewriting it to addition.
+ // Decide on fences then, so a native addition can retain the original
+ // ordering.
+ if (RI->getOperation() == AtomicRMWInst::Sub ||
+ RI->getOperation() == AtomicRMWInst::FSub)
+ return false;
+
+ // Splitting creates several atomics. Use one set of fences around the
+ // whole sequence instead of enforcing ordering on each piece.
+ return true;
+ }
+ Ordering = RI->getOrdering();
+ } else {
+ return false;
}
- return false;
+
+ // Older targets lack atomic memory-order qualifiers, so fences must enforce
+ // any ordering stronger than monotonic.
+ if (!STI.hasMemoryOrdering())
+ return true;
+
+ // Native atomics can carry acquire/release ordering, but PTX has no seq_cst
+ // qualifier. The PTX atomic ABI requires fence.sc followed by atom.acquire.
+ // https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
+ return Ordering == AtomicOrdering::SequentiallyConsistent;
}
AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index b8d55d8446b5c8..1693a565b92da3 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -3366,7 +3366,7 @@ class AtomicSDNodeXForm<SDNode atomic> {
unsigned EltWidth = MN->getMemoryVT().getFixedSizeInBits();
NVPTXMemCacheHintAccess Access{
NVPTXMemCacheHintInstruction::Atom, getAddrSpace(MN),
- /*NumElts=*/1, EltWidth, MN->isVolatile()};
+ /*NumElts=*/1, EltWidth, getMemOrder(MN)};
return getMemCacheHintOperands(MN, Access, SDLoc(N),
/*EmitDiagnostics=*/true)
.EvictionAndPrefetchHint;
@@ -3377,7 +3377,7 @@ class AtomicSDNodeXForm<SDNode atomic> {
unsigned EltWidth = MN->getMemoryVT().getFixedSizeInBits();
NVPTXMemCacheHintAccess Access{
NVPTXMemCacheHintInstruction::Atom, getAddrSpace(MN),
- /*NumElts=*/1, EltWidth, MN->isVolatile()};
+ /*NumElts=*/1, EltWidth, getMemOrder(MN)};
return getMemCacheHintOperands(MN, Access, SDLoc(N),
/*EmitDiagnostics=*/false)
.CachePolicyReg;
@@ -3447,9 +3447,52 @@ defm INT_PTX_ATOM_ADD_64 : F_ATOMIC_2<I64RT, atomic_load_add_i64, "add", "u64",
defm INT_PTX_ATOM_ADD_F16 : F_ATOMIC_2<F16RT, atomic_load_fadd, "add.noftz", "f16", atomic_load_fadd, [SM70, PTX63]>;
defm INT_PTX_ATOM_ADD_BF16 : F_ATOMIC_2<BF16RT, atomic_load_fadd, "add.noftz", "bf16", atomic_load_fadd, [SM90]>;
+defm INT_PTX_ATOM_ADD_F16X2 : F_ATOMIC_2<F16X2RT, atomic_load_fadd, "add.noftz", "f16x2", atomic_load_fadd, [SM60, PTX62]>;
+defm INT_PTX_ATOM_ADD_BF16X2 : F_ATOMIC_2<BF16X2RT, atomic_load_fadd, "add.noftz", "bf16x2", atomic_load_fadd, [SM90, PTX78]>;
defm INT_PTX_ATOM_ADD_F32 : F_ATOMIC_2<F32RT, atomic_load_fadd, "add", "f32", atomic_load_fadd>;
defm INT_PTX_ATOM_ADD_F64 : F_ATOMIC_2<F64RT, atomic_load_fadd, "add", "f64", atomic_load_fadd, [hasAtomAddF64]>;
+// Native vector atomics use one register per f32 element or packed pair of
+// f16/bf16 elements. Two-element f16/bf16 min/max use two 16-bit registers.
+class F_ATOMIC_VECTOR<RegTyInfo T, int N, string Op, string Type> :
+ NVPTXInst<
+ !dag(outs, !listsplat(T.RC, N),
+ !foreach(I, !range(N), "dst" # I)),
+ !con((ins ADDR:$addr),
+ !dag(ins, !listsplat(T.RC, N),
+ !foreach(I, !range(N), "src" # I)),
+ (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
+ EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy)),
+ "atom${sem:sem}${scope:scope}${addsp:addsp}." # Op #
+ "${evictionAndPrefetchHint:l2}.v" # N # "." # Type # " \t{{" #
+ !interleave(!foreach(I, !range(N), "$dst" # I), ", ") #
+ "}}, [$addr], {{" #
+ !interleave(!foreach(I, !range(N), "$src" # I), ", ") #
+ "}}${policy};"> {
+ let mayLoad = true;
+ let mayStore = true;
+ let hasSideEffects = true;
+ let Predicates = [SM90, PTX81];
+}
+
+foreach N = [2, 4] in {
+ def ATOM_ADD_V # N # _F32 : F_ATOMIC_VECTOR<F32RT, N, "add", "f32">;
+ let Predicates = [SM90, PTX94] in
+ def ATOM_ADD_NOFTZ_V # N # _F32 :
+ F_ATOMIC_VECTOR<F32RT, N, "add.noftz", "f32">;
+ foreach T = [F16X2RT, BF16X2RT] in {
+ foreach Op = ["add", "min", "max"] in
+ def ATOM_ # !toupper(Op) # _V # N # _ # !toupper(T.PtxType) :
+ F_ATOMIC_VECTOR<T, N, Op # ".noftz", T.PtxType>;
+ }
+}
+foreach T = [F16RT, BF16RT] in {
+ foreach Op = ["min", "max"] in
+ def ATOM_ # !toupper(Op) # _V2_ # !toupper(T.PtxType) :
+ F_ATOMIC_VECTOR<T, 2, Op # ".noftz", T.PtxType>;
+}
+
// atom_swap
defm INT_PTX_ATOM_SWAP_32 : F_ATOMIC_2<I32RT, atomic_swap_i32, "exch", "b32", atomic_swap>;
defm INT_PTX_ATOM_SWAP_64 : F_ATOMIC_2<I64RT, atomic_swap_i64, "exch", "b64", atomic_swap>;
diff --git a/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.cpp
index dac41465e1bddd..9c2d736884322e 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.cpp
@@ -36,6 +36,9 @@ const char *NVPTXSelectionDAGInfo::getTargetNodeName(unsigned Opcode) const {
MAKE_CASE(NVPTXISD::StoreV2)
MAKE_CASE(NVPTXISD::StoreV4)
MAKE_CASE(NVPTXISD::StoreV8)
+ MAKE_CASE(NVPTXISD::ATOM_VEC_V1)
+ MAKE_CASE(NVPTXISD::ATOM_VEC_V2)
+ MAKE_CASE(NVPTXISD::ATOM_VEC_V4)
MAKE_CASE(NVPTXISD::SETP_F16X2)
MAKE_CASE(NVPTXISD::SETP_BF16X2)
MAKE_CASE(NVPTXISD::UNPACK_VECTOR)
diff --git a/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h b/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h
index 9dd0a1eaa5856b..b345570c23cb6c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h
@@ -42,7 +42,14 @@ enum NodeType : unsigned {
StoreV2,
StoreV4,
StoreV8,
- LAST_MEMORY_OPCODE = StoreV8,
+
+ /// Native vector atomicrmw nodes. Each takes (chain, ptr, val0..valN-1,
+ /// op_code) and produces (res0..resN-1, chain), with one value per PTX
+ /// register. op_code is 0=fadd, 1=fmin, or 2=fmax.
+ ATOM_VEC_V1,
+ ATOM_VEC_V2,
+ ATOM_VEC_V4,
+ LAST_MEMORY_OPCODE = ATOM_VEC_V4,
};
} // namespace NVPTXISD
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
index 5b631504b96257..fb2eb357bb6d65 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
@@ -22,14 +22,14 @@ define <1 x i8> @xchg_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b8 %r5, [xchg_acq_rel_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
-; SM60-NEXT: and.b32 %r7, %r6, 3;
-; SM60-NEXT: shl.b32 %r1, %r7, 3;
-; SM60-NEXT: shl.b32 %r8, 255, %r1;
-; SM60-NEXT: not.b32 %r2, %r8;
-; SM60-NEXT: shl.b32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: shl.b32 %r7, 255, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.param.b8 %r8, [xchg_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r3, %r8, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r12, [%rd1];
; SM60-NEXT: $L__BB0_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
@@ -128,8 +128,8 @@ define <1 x i16> @xchg_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i16_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
; SM60-NEXT: and.b32 %r7, %r6, 3;
@@ -225,8 +225,8 @@ define <1 x i32> @xchg_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: atom.cta.global.exch.b32 %r2, [%rd1], %r1;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
@@ -325,8 +325,8 @@ define <1 x i64> @xchg_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i64_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: atom.cta.global.exch.b64 %rd3, [%rd1], %rd2;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
@@ -415,14 +415,14 @@ define <1 x i8> @add_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b8 %r6, [add_acq_rel_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: and.b32 %r8, %r7, 3;
-; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
; SM60-NEXT: shl.b32 %r2, 255, %r1;
; SM60-NEXT: not.b32 %r3, %r2;
-; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.param.b8 %r8, [add_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r4, %r8, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
; SM60-NEXT: $L__BB16_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
@@ -1014,14 +1014,14 @@ define <1 x i8> @sub_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b8 %r6, [sub_acq_rel_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: and.b32 %r8, %r7, 3;
-; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
; SM60-NEXT: shl.b32 %r2, 255, %r1;
; SM60-NEXT: not.b32 %r3, %r2;
-; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.param.b8 %r8, [sub_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r4, %r8, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
; SM60-NEXT: $L__BB32_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
@@ -1358,74 +1358,90 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-LABEL: sub_acq_rel_v8i16_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<3>;
-; SM60-NEXT: .reg .b16 %rs<25>;
-; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b16 %rs<41>;
+; SM60-NEXT: .reg .b32 %r<17>;
; SM60-NEXT: .reg .b64 %rd<22>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r6;
+; SM60-NEXT: neg.s16 %rs3, %rs2;
+; SM60-NEXT: neg.s16 %rs4, %rs1;
+; SM60-NEXT: mov.b32 %r2, {%rs4, %rs3};
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r5;
+; SM60-NEXT: neg.s16 %rs7, %rs6;
+; SM60-NEXT: neg.s16 %rs8, %rs5;
+; SM60-NEXT: mov.b32 %r1, {%rs8, %rs7};
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r8;
+; SM60-NEXT: neg.s16 %rs11, %rs10;
+; SM60-NEXT: neg.s16 %rs12, %rs9;
+; SM60-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r7;
+; SM60-NEXT: neg.s16 %rs15, %rs14;
+; SM60-NEXT: neg.s16 %rs16, %rs13;
+; SM60-NEXT: mov.b32 %r3, {%rs16, %rs15};
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i16_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r1;
+; SM60-NEXT: mov.b32 {%rs23, %rs24}, %r2;
; SM60-NEXT: $L__BB39_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
-; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
-; SM60-NEXT: sub.s16 %rs6, %rs3, %rs1;
-; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
-; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r10;
-; SM60-NEXT: sub.s16 %rs11, %rs10, %rs8;
-; SM60-NEXT: sub.s16 %rs12, %rs9, %rs7;
-; SM60-NEXT: mov.b32 %r6, {%rs12, %rs11};
-; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r13;
+; SM60-NEXT: add.s16 %rs21, %rs20, %rs18;
+; SM60-NEXT: add.s16 %rs22, %rs19, %rs17;
+; SM60-NEXT: mov.b32 %r9, {%rs22, %rs21};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: mov.b32 {%rs25, %rs26}, %r14;
+; SM60-NEXT: add.s16 %rs27, %rs26, %rs24;
+; SM60-NEXT: add.s16 %rs28, %rs25, %rs23;
+; SM60-NEXT: mov.b32 %r10, {%rs28, %rs27};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r14;
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
; SM60-NEXT: @%p1 bra $L__BB39_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
-; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM60-NEXT: mov.b32 {%rs29, %rs30}, %r3;
+; SM60-NEXT: mov.b32 {%rs35, %rs36}, %r4;
; SM60-NEXT: $L__BB39_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
-; SM60-NEXT: sub.s16 %rs17, %rs16, %rs14;
-; SM60-NEXT: sub.s16 %rs18, %rs15, %rs13;
-; SM60-NEXT: mov.b32 %r7, {%rs18, %rs17};
-; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
-; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
-; SM60-NEXT: sub.s16 %rs23, %rs22, %rs20;
-; SM60-NEXT: sub.s16 %rs24, %rs21, %rs19;
-; SM60-NEXT: mov.b32 %r8, {%rs24, %rs23};
-; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: mov.b32 {%rs31, %rs32}, %r15;
+; SM60-NEXT: add.s16 %rs33, %rs32, %rs30;
+; SM60-NEXT: add.s16 %rs34, %rs31, %rs29;
+; SM60-NEXT: mov.b32 %r11, {%rs34, %rs33};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: mov.b32 {%rs37, %rs38}, %r16;
+; SM60-NEXT: add.s16 %rs39, %rs38, %rs36;
+; SM60-NEXT: add.s16 %rs40, %rs37, %rs35;
+; SM60-NEXT: mov.b32 %r12, {%rs40, %rs39};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r16;
; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
-; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd21;
; SM60-NEXT: @%p2 bra $L__BB39_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -1439,9 +1455,9 @@ define <1 x i32> @sub_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [sub_acq_rel_v1i32_global_cta_param_1];
; SM60-NEXT: neg.s32 %r2, %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r2;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r3;
@@ -1458,14 +1474,14 @@ define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32>
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v2i32_global_cta_param_1];
-; SM60-NEXT: neg.s32 %r3, %r1;
-; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1], %r3;
-; SM60-NEXT: neg.s32 %r5, %r2;
-; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r5;
+; SM60-NEXT: neg.s32 %r3, %r2;
+; SM60-NEXT: neg.s32 %r4, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r4;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r3;
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -1479,18 +1495,18 @@ define <4 x i32> @sub_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v4i32_global_cta_param_1];
-; SM60-NEXT: neg.s32 %r5, %r1;
-; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1], %r5;
+; SM60-NEXT: neg.s32 %r5, %r4;
+; SM60-NEXT: neg.s32 %r6, %r3;
; SM60-NEXT: neg.s32 %r7, %r2;
-; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+4], %r7;
-; SM60-NEXT: neg.s32 %r9, %r3;
-; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+8], %r9;
-; SM60-NEXT: neg.s32 %r11, %r4;
-; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r11;
+; SM60-NEXT: neg.s32 %r8, %r1;
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r5;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -1504,28 +1520,28 @@ define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i32_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i32_global_cta_param_1];
-; SM60-NEXT: neg.s32 %r9, %r5;
-; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1], %r9;
-; SM60-NEXT: neg.s32 %r11, %r6;
-; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+4], %r11;
-; SM60-NEXT: neg.s32 %r13, %r7;
-; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+8], %r13;
-; SM60-NEXT: neg.s32 %r15, %r8;
-; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+12], %r15;
-; SM60-NEXT: neg.s32 %r17, %r1;
-; SM60-NEXT: atom.cta.global.add.u32 %r18, [%rd1+16], %r17;
-; SM60-NEXT: neg.s32 %r19, %r2;
-; SM60-NEXT: atom.cta.global.add.u32 %r20, [%rd1+20], %r19;
-; SM60-NEXT: neg.s32 %r21, %r3;
-; SM60-NEXT: atom.cta.global.add.u32 %r22, [%rd1+24], %r21;
-; SM60-NEXT: neg.s32 %r23, %r4;
-; SM60-NEXT: atom.cta.global.add.u32 %r24, [%rd1+28], %r23;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM60-NEXT: neg.s32 %r5, %r4;
+; SM60-NEXT: neg.s32 %r6, %r3;
+; SM60-NEXT: neg.s32 %r7, %r2;
+; SM60-NEXT: neg.s32 %r8, %r1;
+; SM60-NEXT: ld.param.v4.b32 {%r9, %r10, %r11, %r12}, [sub_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: neg.s32 %r13, %r12;
+; SM60-NEXT: neg.s32 %r14, %r11;
+; SM60-NEXT: neg.s32 %r15, %r10;
+; SM60-NEXT: neg.s32 %r16, %r9;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: atom.cta.global.add.u32 %r17, [%rd1], %r16;
+; SM60-NEXT: atom.cta.global.add.u32 %r18, [%rd1+4], %r15;
+; SM60-NEXT: atom.cta.global.add.u32 %r19, [%rd1+8], %r14;
+; SM60-NEXT: atom.cta.global.add.u32 %r20, [%rd1+12], %r13;
+; SM60-NEXT: atom.cta.global.add.u32 %r21, [%rd1+16], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r22, [%rd1+20], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r23, [%rd1+24], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r24, [%rd1+28], %r5;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -1538,9 +1554,9 @@ define <1 x i64> @sub_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i64_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i64_global_cta_param_1];
; SM60-NEXT: neg.s64 %rd3, %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: atom.cta.global.add.u64 %rd4, [%rd1], %rd3;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd4;
@@ -1556,14 +1572,14 @@ define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64>
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i64_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v2i64_global_cta_param_1];
-; SM60-NEXT: neg.s64 %rd4, %rd2;
-; SM60-NEXT: atom.cta.global.add.u64 %rd5, [%rd1], %rd4;
-; SM60-NEXT: neg.s64 %rd6, %rd3;
-; SM60-NEXT: atom.cta.global.add.u64 %rd7, [%rd1+8], %rd6;
+; SM60-NEXT: neg.s64 %rd4, %rd3;
+; SM60-NEXT: neg.s64 %rd5, %rd2;
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd7};
+; SM60-NEXT: atom.cta.global.add.u64 %rd6, [%rd1], %rd5;
+; SM60-NEXT: atom.cta.global.add.u64 %rd7, [%rd1+8], %rd4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -1576,20 +1592,20 @@ define <4 x i64> @sub_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64>
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i64_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v4i64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v4i64_global_cta_param_1];
-; SM60-NEXT: neg.s64 %rd6, %rd4;
-; SM60-NEXT: atom.cta.global.add.u64 %rd7, [%rd1], %rd6;
-; SM60-NEXT: neg.s64 %rd8, %rd5;
-; SM60-NEXT: atom.cta.global.add.u64 %rd9, [%rd1+8], %rd8;
-; SM60-NEXT: neg.s64 %rd10, %rd2;
-; SM60-NEXT: atom.cta.global.add.u64 %rd11, [%rd1+16], %rd10;
-; SM60-NEXT: neg.s64 %rd12, %rd3;
-; SM60-NEXT: atom.cta.global.add.u64 %rd13, [%rd1+24], %rd12;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd11, %rd13};
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd7, %rd9};
+; SM60-NEXT: neg.s64 %rd4, %rd3;
+; SM60-NEXT: neg.s64 %rd5, %rd2;
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: neg.s64 %rd8, %rd7;
+; SM60-NEXT: neg.s64 %rd9, %rd6;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: atom.cta.global.add.u64 %rd10, [%rd1], %rd9;
+; SM60-NEXT: atom.cta.global.add.u64 %rd11, [%rd1+8], %rd8;
+; SM60-NEXT: atom.cta.global.add.u64 %rd12, [%rd1+16], %rd5;
+; SM60-NEXT: atom.cta.global.add.u64 %rd13, [%rd1+24], %rd4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -1602,32 +1618,32 @@ define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i64_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v8i64_global_cta_param_1+48];
-; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v8i64_global_cta_param_1+32];
-; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [sub_acq_rel_v8i64_global_cta_param_1];
-; SM60-NEXT: neg.s64 %rd10, %rd8;
-; SM60-NEXT: atom.cta.global.add.u64 %rd11, [%rd1], %rd10;
-; SM60-NEXT: neg.s64 %rd12, %rd9;
-; SM60-NEXT: atom.cta.global.add.u64 %rd13, [%rd1+8], %rd12;
-; SM60-NEXT: neg.s64 %rd14, %rd6;
-; SM60-NEXT: atom.cta.global.add.u64 %rd15, [%rd1+16], %rd14;
-; SM60-NEXT: neg.s64 %rd16, %rd7;
-; SM60-NEXT: atom.cta.global.add.u64 %rd17, [%rd1+24], %rd16;
-; SM60-NEXT: neg.s64 %rd18, %rd4;
-; SM60-NEXT: atom.cta.global.add.u64 %rd19, [%rd1+32], %rd18;
-; SM60-NEXT: neg.s64 %rd20, %rd5;
-; SM60-NEXT: atom.cta.global.add.u64 %rd21, [%rd1+40], %rd20;
-; SM60-NEXT: neg.s64 %rd22, %rd2;
-; SM60-NEXT: atom.cta.global.add.u64 %rd23, [%rd1+48], %rd22;
-; SM60-NEXT: neg.s64 %rd24, %rd3;
-; SM60-NEXT: atom.cta.global.add.u64 %rd25, [%rd1+56], %rd24;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd23, %rd25};
-; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd19, %rd21};
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd15, %rd17};
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd11, %rd13};
+; SM60-NEXT: neg.s64 %rd4, %rd3;
+; SM60-NEXT: neg.s64 %rd5, %rd2;
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: neg.s64 %rd8, %rd7;
+; SM60-NEXT: neg.s64 %rd9, %rd6;
+; SM60-NEXT: ld.param.v2.b64 {%rd10, %rd11}, [sub_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: neg.s64 %rd12, %rd11;
+; SM60-NEXT: neg.s64 %rd13, %rd10;
+; SM60-NEXT: ld.param.v2.b64 {%rd14, %rd15}, [sub_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: neg.s64 %rd16, %rd15;
+; SM60-NEXT: neg.s64 %rd17, %rd14;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: atom.cta.global.add.u64 %rd18, [%rd1], %rd17;
+; SM60-NEXT: atom.cta.global.add.u64 %rd19, [%rd1+8], %rd16;
+; SM60-NEXT: atom.cta.global.add.u64 %rd20, [%rd1+16], %rd13;
+; SM60-NEXT: atom.cta.global.add.u64 %rd21, [%rd1+24], %rd12;
+; SM60-NEXT: atom.cta.global.add.u64 %rd22, [%rd1+32], %rd9;
+; SM60-NEXT: atom.cta.global.add.u64 %rd23, [%rd1+40], %rd8;
+; SM60-NEXT: atom.cta.global.add.u64 %rd24, [%rd1+48], %rd5;
+; SM60-NEXT: atom.cta.global.add.u64 %rd25, [%rd1+56], %rd4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd24, %rd25};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd22, %rd23};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd20, %rd21};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -1642,17 +1658,17 @@ define <1 x i8> @and_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b8 %r1, [and_acq_rel_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd2, %rd1, -4;
-; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
-; SM60-NEXT: and.b32 %r3, %r2, 3;
-; SM60-NEXT: shl.b32 %r4, %r3, 3;
-; SM60-NEXT: shl.b32 %r5, 255, %r4;
-; SM60-NEXT: not.b32 %r6, %r5;
-; SM60-NEXT: shl.b32 %r7, %r1, %r4;
-; SM60-NEXT: or.b32 %r8, %r7, %r6;
+; SM60-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM60-NEXT: and.b32 %r2, %r1, 3;
+; SM60-NEXT: shl.b32 %r3, %r2, 3;
+; SM60-NEXT: shl.b32 %r4, 255, %r3;
+; SM60-NEXT: not.b32 %r5, %r4;
+; SM60-NEXT: ld.param.b8 %r6, [and_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r7, %r6, %r3;
+; SM60-NEXT: or.b32 %r8, %r7, %r5;
; SM60-NEXT: atom.cta.global.and.b32 %r9, [%rd2], %r8;
-; SM60-NEXT: shr.u32 %r10, %r9, %r4;
+; SM60-NEXT: shr.u32 %r10, %r9, %r3;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b8 [func_retval0], %r10;
; SM60-NEXT: ret;
@@ -2088,14 +2104,14 @@ define <1 x i8> @nand_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b8 %r6, [nand_acq_rel_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: and.b32 %r8, %r7, 3;
-; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
; SM60-NEXT: shl.b32 %r2, 255, %r1;
; SM60-NEXT: not.b32 %r3, %r2;
-; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.param.b8 %r8, [nand_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r4, %r8, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
; SM60-NEXT: $L__BB64_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
@@ -2405,21 +2421,21 @@ define <1 x i32> @nand_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_acq_rel_v1i32_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i32_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b32 %r1, [nand_acq_rel_v1i32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
; SM60-NEXT: $L__BB72_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
; SM60-NEXT: not.b32 %r4, %r3;
-; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
; SM60-NEXT: @%p1 bra $L__BB72_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -2642,21 +2658,21 @@ define <1 x i64> @nand_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM60-NEXT: .reg .b64 %rd<7>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd3];
+; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd2];
; SM60-NEXT: $L__BB76_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b64 %rd4, %rd6, %rd1;
+; SM60-NEXT: and.b64 %rd4, %rd6, %rd3;
; SM60-NEXT: not.b64 %rd5, %rd4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
-; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
-; SM60-NEXT: mov.b64 %rd6, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5;
+; SM60-NEXT: setp.ne.b64 %p1, %rd1, %rd6;
+; SM60-NEXT: mov.b64 %rd6, %rd1;
; SM60-NEXT: @%p1 bra $L__BB76_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -2671,29 +2687,29 @@ define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd5, [nand_acq_rel_v2i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [nand_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v2i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
; SM60-NEXT: $L__BB77_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd3, %rd10;
-; SM60-NEXT: and.b64 %rd6, %rd3, %rd2;
+; SM60-NEXT: and.b64 %rd6, %rd10, %rd1;
; SM60-NEXT: not.b64 %rd7, %rd6;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd10, [%rd5], %rd3, %rd7;
-; SM60-NEXT: setp.ne.b64 %p1, %rd10, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
+; SM60-NEXT: mov.b64 %rd10, %rd3;
; SM60-NEXT: @%p1 bra $L__BB77_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
-; SM60-NEXT: $L__BB77_3: // %atomicrmw.start4
+; SM60-NEXT: $L__BB77_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b64 %rd8, %rd11, %rd1;
+; SM60-NEXT: and.b64 %rd8, %rd11, %rd2;
; SM60-NEXT: not.b64 %rd9, %rd8;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
; SM60-NEXT: mov.b64 %rd11, %rd4;
; SM60-NEXT: @%p2 bra $L__BB77_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end3
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd4};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -2708,50 +2724,50 @@ define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd9, [nand_acq_rel_v4i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [nand_acq_rel_v4i64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v4i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
; SM60-NEXT: $L__BB78_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd4, %rd18;
-; SM60-NEXT: and.b64 %rd10, %rd4, %rd3;
+; SM60-NEXT: mov.b64 %rd5, %rd18;
+; SM60-NEXT: and.b64 %rd10, %rd5, %rd3;
; SM60-NEXT: not.b64 %rd11, %rd10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd4, %rd11;
-; SM60-NEXT: setp.ne.b64 %p1, %rd18, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
; SM60-NEXT: @%p1 bra $L__BB78_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
-; SM60-NEXT: $L__BB78_3: // %atomicrmw.start8
+; SM60-NEXT: $L__BB78_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd19;
-; SM60-NEXT: and.b64 %rd12, %rd5, %rd2;
+; SM60-NEXT: mov.b64 %rd6, %rd19;
+; SM60-NEXT: and.b64 %rd12, %rd6, %rd4;
; SM60-NEXT: not.b64 %rd13, %rd12;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd5, %rd13;
-; SM60-NEXT: setp.ne.b64 %p2, %rd19, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM60-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
; SM60-NEXT: @%p2 bra $L__BB78_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end7
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
-; SM60-NEXT: $L__BB78_5: // %atomicrmw.start20
+; SM60-NEXT: $L__BB78_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd7, %rd20;
-; SM60-NEXT: and.b64 %rd14, %rd7, %rd1;
+; SM60-NEXT: and.b64 %rd14, %rd20, %rd1;
; SM60-NEXT: not.b64 %rd15, %rd14;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd20, [%rd9+16], %rd7, %rd15;
-; SM60-NEXT: setp.ne.b64 %p3, %rd20, %rd7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
+; SM60-NEXT: mov.b64 %rd20, %rd7;
; SM60-NEXT: @%p3 bra $L__BB78_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
-; SM60-NEXT: $L__BB78_7: // %atomicrmw.start28
+; SM60-NEXT: $L__BB78_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b64 %rd16, %rd21, %rd6;
+; SM60-NEXT: and.b64 %rd16, %rd21, %rd2;
; SM60-NEXT: not.b64 %rd17, %rd16;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
; SM60-NEXT: mov.b64 %rd21, %rd8;
; SM60-NEXT: @%p4 bra $L__BB78_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end27
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd20, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
@@ -2767,92 +2783,92 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd17, [nand_acq_rel_v8i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [nand_acq_rel_v8i64_global_cta_param_1+48];
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [nand_acq_rel_v8i64_global_cta_param_1+32];
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [nand_acq_rel_v8i64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [nand_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [nand_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [nand_acq_rel_v8i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
; SM60-NEXT: $L__BB79_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd34;
-; SM60-NEXT: and.b64 %rd18, %rd5, %rd4;
+; SM60-NEXT: mov.b64 %rd7, %rd34;
+; SM60-NEXT: and.b64 %rd18, %rd7, %rd5;
; SM60-NEXT: not.b64 %rd19, %rd18;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd5, %rd19;
-; SM60-NEXT: setp.ne.b64 %p1, %rd34, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM60-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
; SM60-NEXT: @%p1 bra $L__BB79_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
-; SM60-NEXT: $L__BB79_3: // %atomicrmw.start12
+; SM60-NEXT: $L__BB79_3: // %atomicrmw.start10
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd6, %rd35;
-; SM60-NEXT: and.b64 %rd20, %rd6, %rd3;
+; SM60-NEXT: mov.b64 %rd8, %rd35;
+; SM60-NEXT: and.b64 %rd20, %rd8, %rd6;
; SM60-NEXT: not.b64 %rd21, %rd20;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd6, %rd21;
-; SM60-NEXT: setp.ne.b64 %p2, %rd35, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM60-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
; SM60-NEXT: @%p2 bra $L__BB79_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end11
+; SM60-NEXT: // %bb.4: // %atomicrmw.end9
; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
-; SM60-NEXT: $L__BB79_5: // %atomicrmw.start24
+; SM60-NEXT: $L__BB79_5: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd8, %rd36;
-; SM60-NEXT: and.b64 %rd22, %rd8, %rd2;
+; SM60-NEXT: mov.b64 %rd9, %rd36;
+; SM60-NEXT: and.b64 %rd22, %rd9, %rd3;
; SM60-NEXT: not.b64 %rd23, %rd22;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd8, %rd23;
-; SM60-NEXT: setp.ne.b64 %p3, %rd36, %rd8;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM60-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
; SM60-NEXT: @%p3 bra $L__BB79_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end23
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
-; SM60-NEXT: $L__BB79_7: // %atomicrmw.start32
+; SM60-NEXT: $L__BB79_7: // %atomicrmw.start26
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd9, %rd37;
-; SM60-NEXT: and.b64 %rd24, %rd9, %rd7;
+; SM60-NEXT: mov.b64 %rd10, %rd37;
+; SM60-NEXT: and.b64 %rd24, %rd10, %rd4;
; SM60-NEXT: not.b64 %rd25, %rd24;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd9, %rd25;
-; SM60-NEXT: setp.ne.b64 %p4, %rd37, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM60-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
; SM60-NEXT: @%p4 bra $L__BB79_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end31
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
-; SM60-NEXT: $L__BB79_9: // %atomicrmw.start48
+; SM60-NEXT: $L__BB79_9: // %atomicrmw.start40
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd12, %rd38;
-; SM60-NEXT: and.b64 %rd26, %rd12, %rd1;
+; SM60-NEXT: mov.b64 %rd13, %rd38;
+; SM60-NEXT: and.b64 %rd26, %rd13, %rd1;
; SM60-NEXT: not.b64 %rd27, %rd26;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd12, %rd27;
-; SM60-NEXT: setp.ne.b64 %p5, %rd38, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM60-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
; SM60-NEXT: @%p5 bra $L__BB79_9;
-; SM60-NEXT: // %bb.10: // %atomicrmw.end47
+; SM60-NEXT: // %bb.10: // %atomicrmw.end39
; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
-; SM60-NEXT: $L__BB79_11: // %atomicrmw.start56
+; SM60-NEXT: $L__BB79_11: // %atomicrmw.start46
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd13, %rd39;
-; SM60-NEXT: and.b64 %rd28, %rd13, %rd11;
+; SM60-NEXT: mov.b64 %rd14, %rd39;
+; SM60-NEXT: and.b64 %rd28, %rd14, %rd2;
; SM60-NEXT: not.b64 %rd29, %rd28;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd13, %rd29;
-; SM60-NEXT: setp.ne.b64 %p6, %rd39, %rd13;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM60-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
; SM60-NEXT: @%p6 bra $L__BB79_11;
-; SM60-NEXT: // %bb.12: // %atomicrmw.end55
+; SM60-NEXT: // %bb.12: // %atomicrmw.end45
; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
-; SM60-NEXT: $L__BB79_13: // %atomicrmw.start68
+; SM60-NEXT: $L__BB79_13: // %atomicrmw.start56
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd15, %rd40;
-; SM60-NEXT: and.b64 %rd30, %rd15, %rd10;
+; SM60-NEXT: and.b64 %rd30, %rd40, %rd11;
; SM60-NEXT: not.b64 %rd31, %rd30;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd40, [%rd17+48], %rd15, %rd31;
-; SM60-NEXT: setp.ne.b64 %p7, %rd40, %rd15;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
+; SM60-NEXT: mov.b64 %rd40, %rd15;
; SM60-NEXT: @%p7 bra $L__BB79_13;
-; SM60-NEXT: // %bb.14: // %atomicrmw.end67
+; SM60-NEXT: // %bb.14: // %atomicrmw.end55
; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
-; SM60-NEXT: $L__BB79_15: // %atomicrmw.start76
+; SM60-NEXT: $L__BB79_15: // %atomicrmw.start62
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b64 %rd32, %rd41, %rd14;
+; SM60-NEXT: and.b64 %rd32, %rd41, %rd12;
; SM60-NEXT: not.b64 %rd33, %rd32;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
; SM60-NEXT: mov.b64 %rd41, %rd16;
; SM60-NEXT: @%p8 bra $L__BB79_15;
-; SM60-NEXT: // %bb.16: // %atomicrmw.end75
+; SM60-NEXT: // %bb.16: // %atomicrmw.end61
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd40, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
@@ -2870,14 +2886,14 @@ define <1 x i8> @or_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %va
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b8 %r1, [or_acq_rel_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd2, %rd1, -4;
-; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
-; SM60-NEXT: and.b32 %r3, %r2, 3;
-; SM60-NEXT: shl.b32 %r4, %r3, 3;
-; SM60-NEXT: shl.b32 %r5, %r1, %r4;
+; SM60-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM60-NEXT: and.b32 %r2, %r1, 3;
+; SM60-NEXT: shl.b32 %r3, %r2, 3;
+; SM60-NEXT: ld.param.b8 %r4, [or_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r5, %r4, %r3;
; SM60-NEXT: atom.cta.global.or.b32 %r6, [%rd2], %r5;
-; SM60-NEXT: shr.u32 %r7, %r6, %r4;
+; SM60-NEXT: shr.u32 %r7, %r6, %r3;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b8 [func_retval0], %r7;
; SM60-NEXT: ret;
@@ -3306,14 +3322,14 @@ define <1 x i8> @xor_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b8 %r1, [xor_acq_rel_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd2, %rd1, -4;
-; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
-; SM60-NEXT: and.b32 %r3, %r2, 3;
-; SM60-NEXT: shl.b32 %r4, %r3, 3;
-; SM60-NEXT: shl.b32 %r5, %r1, %r4;
+; SM60-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM60-NEXT: and.b32 %r2, %r1, 3;
+; SM60-NEXT: shl.b32 %r3, %r2, 3;
+; SM60-NEXT: ld.param.b8 %r4, [xor_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r5, %r4, %r3;
; SM60-NEXT: atom.cta.global.xor.b32 %r6, [%rd2], %r5;
-; SM60-NEXT: shr.u32 %r7, %r6, %r4;
+; SM60-NEXT: shr.u32 %r7, %r6, %r3;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b8 [func_retval0], %r7;
; SM60-NEXT: ret;
@@ -3737,14 +3753,14 @@ define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM60-LABEL: max_acq_rel_v1i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b16 %rs<5>;
; SM60-NEXT: .reg .b32 %r<15>;
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.s8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NEXT: and.b32 %r5, %r4, 3;
@@ -3752,12 +3768,13 @@ define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM60-NEXT: shl.b32 %r6, 255, %r1;
; SM60-NEXT: not.b32 %r2, %r6;
; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: cvt.s16.s8 %rs3, %rs1;
; SM60-NEXT: $L__BB112_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r7, %r14, %r1;
; SM60-NEXT: cvt.s8.s32 %rs2, %r7;
-; SM60-NEXT: max.s16 %rs3, %rs2, %rs1;
-; SM60-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM60-NEXT: max.s16 %rs4, %rs2, %rs3;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs4;
; SM60-NEXT: and.b32 %r9, %r8, 255;
; SM60-NEXT: shl.b32 %r10, %r9, %r1;
; SM60-NEXT: and.b32 %r11, %r14, %r2;
@@ -4004,9 +4021,9 @@ define <1 x i16> @max_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i16_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NEXT: and.b32 %r5, %r4, 3;
@@ -4369,14 +4386,14 @@ define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM60-LABEL: min_acq_rel_v1i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b16 %rs<5>;
; SM60-NEXT: .reg .b32 %r<15>;
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.s8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NEXT: and.b32 %r5, %r4, 3;
@@ -4384,12 +4401,13 @@ define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM60-NEXT: shl.b32 %r6, 255, %r1;
; SM60-NEXT: not.b32 %r2, %r6;
; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: cvt.s16.s8 %rs3, %rs1;
; SM60-NEXT: $L__BB128_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r7, %r14, %r1;
; SM60-NEXT: cvt.s8.s32 %rs2, %r7;
-; SM60-NEXT: min.s16 %rs3, %rs2, %rs1;
-; SM60-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM60-NEXT: min.s16 %rs4, %rs2, %rs3;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs4;
; SM60-NEXT: and.b32 %r9, %r8, 255;
; SM60-NEXT: shl.b32 %r10, %r9, %r1;
; SM60-NEXT: and.b32 %r11, %r14, %r2;
@@ -4636,9 +4654,9 @@ define <1 x i16> @min_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i16_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NEXT: and.b32 %r5, %r4, 3;
@@ -5243,9 +5261,9 @@ define <1 x i16> @umax_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i16_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NEXT: and.b32 %r5, %r4, 3;
@@ -5850,9 +5868,9 @@ define <1 x i16> @umin_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i16_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NEXT: and.b32 %r5, %r4, 3;
@@ -6476,9 +6494,9 @@ define <1 x i16> @uinc_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i16_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NEXT: and.b32 %r5, %r4, 3;
@@ -6787,22 +6805,22 @@ define <1 x i64> @uinc_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM60-NEXT: .reg .b64 %rd<7>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd3];
+; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd2];
; SM60-NEXT: $L__BB188_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd4, %rd6, 1;
-; SM60-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM60-NEXT: setp.ge.u64 %p1, %rd6, %rd3;
; SM60-NEXT: selp.b64 %rd5, 0, %rd4, %p1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
-; SM60-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
-; SM60-NEXT: mov.b64 %rd6, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5;
+; SM60-NEXT: setp.ne.b64 %p2, %rd1, %rd6;
+; SM60-NEXT: mov.b64 %rd6, %rd1;
; SM60-NEXT: @%p2 bra $L__BB188_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -6817,31 +6835,31 @@ define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd5, [uinc_wrap_acq_rel_v2i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
; SM60-NEXT: $L__BB189_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd3, %rd10;
-; SM60-NEXT: add.s64 %rd6, %rd3, 1;
-; SM60-NEXT: setp.ge.u64 %p1, %rd3, %rd2;
+; SM60-NEXT: add.s64 %rd6, %rd10, 1;
+; SM60-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
; SM60-NEXT: selp.b64 %rd7, 0, %rd6, %p1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd10, [%rd5], %rd3, %rd7;
-; SM60-NEXT: setp.ne.b64 %p2, %rd10, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM60-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
+; SM60-NEXT: mov.b64 %rd10, %rd3;
; SM60-NEXT: @%p2 bra $L__BB189_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
-; SM60-NEXT: $L__BB189_3: // %atomicrmw.start4
+; SM60-NEXT: $L__BB189_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd8, %rd11, 1;
-; SM60-NEXT: setp.ge.u64 %p3, %rd11, %rd1;
+; SM60-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
; SM60-NEXT: selp.b64 %rd9, 0, %rd8, %p3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM60-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
; SM60-NEXT: mov.b64 %rd11, %rd4;
; SM60-NEXT: @%p4 bra $L__BB189_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end3
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd4};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -6856,54 +6874,54 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd9, [uinc_wrap_acq_rel_v4i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
; SM60-NEXT: $L__BB190_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd4, %rd18;
-; SM60-NEXT: add.s64 %rd10, %rd4, 1;
-; SM60-NEXT: setp.ge.u64 %p1, %rd4, %rd3;
+; SM60-NEXT: mov.b64 %rd5, %rd18;
+; SM60-NEXT: add.s64 %rd10, %rd5, 1;
+; SM60-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
; SM60-NEXT: selp.b64 %rd11, 0, %rd10, %p1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd4, %rd11;
-; SM60-NEXT: setp.ne.b64 %p2, %rd18, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
; SM60-NEXT: @%p2 bra $L__BB190_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
-; SM60-NEXT: $L__BB190_3: // %atomicrmw.start8
+; SM60-NEXT: $L__BB190_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd19;
-; SM60-NEXT: add.s64 %rd12, %rd5, 1;
-; SM60-NEXT: setp.ge.u64 %p3, %rd5, %rd2;
+; SM60-NEXT: mov.b64 %rd6, %rd19;
+; SM60-NEXT: add.s64 %rd12, %rd6, 1;
+; SM60-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
; SM60-NEXT: selp.b64 %rd13, 0, %rd12, %p3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd5, %rd13;
-; SM60-NEXT: setp.ne.b64 %p4, %rd19, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
; SM60-NEXT: @%p4 bra $L__BB190_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end7
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
-; SM60-NEXT: $L__BB190_5: // %atomicrmw.start20
+; SM60-NEXT: $L__BB190_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd7, %rd20;
-; SM60-NEXT: add.s64 %rd14, %rd7, 1;
-; SM60-NEXT: setp.ge.u64 %p5, %rd7, %rd1;
+; SM60-NEXT: add.s64 %rd14, %rd20, 1;
+; SM60-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
; SM60-NEXT: selp.b64 %rd15, 0, %rd14, %p5;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd20, [%rd9+16], %rd7, %rd15;
-; SM60-NEXT: setp.ne.b64 %p6, %rd20, %rd7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM60-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
+; SM60-NEXT: mov.b64 %rd20, %rd7;
; SM60-NEXT: @%p6 bra $L__BB190_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
-; SM60-NEXT: $L__BB190_7: // %atomicrmw.start28
+; SM60-NEXT: $L__BB190_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd16, %rd21, 1;
-; SM60-NEXT: setp.ge.u64 %p7, %rd21, %rd6;
+; SM60-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
; SM60-NEXT: selp.b64 %rd17, 0, %rd16, %p7;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM60-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
; SM60-NEXT: mov.b64 %rd21, %rd8;
; SM60-NEXT: @%p8 bra $L__BB190_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end27
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd20, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
@@ -6919,100 +6937,100 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd17, [uinc_wrap_acq_rel_v8i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+48];
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+32];
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
; SM60-NEXT: $L__BB191_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd34;
-; SM60-NEXT: add.s64 %rd18, %rd5, 1;
-; SM60-NEXT: setp.ge.u64 %p1, %rd5, %rd4;
+; SM60-NEXT: mov.b64 %rd7, %rd34;
+; SM60-NEXT: add.s64 %rd18, %rd7, 1;
+; SM60-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
; SM60-NEXT: selp.b64 %rd19, 0, %rd18, %p1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd5, %rd19;
-; SM60-NEXT: setp.ne.b64 %p2, %rd34, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
; SM60-NEXT: @%p2 bra $L__BB191_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
-; SM60-NEXT: $L__BB191_3: // %atomicrmw.start12
+; SM60-NEXT: $L__BB191_3: // %atomicrmw.start10
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd6, %rd35;
-; SM60-NEXT: add.s64 %rd20, %rd6, 1;
-; SM60-NEXT: setp.ge.u64 %p3, %rd6, %rd3;
+; SM60-NEXT: mov.b64 %rd8, %rd35;
+; SM60-NEXT: add.s64 %rd20, %rd8, 1;
+; SM60-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
; SM60-NEXT: selp.b64 %rd21, 0, %rd20, %p3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd6, %rd21;
-; SM60-NEXT: setp.ne.b64 %p4, %rd35, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
; SM60-NEXT: @%p4 bra $L__BB191_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end11
+; SM60-NEXT: // %bb.4: // %atomicrmw.end9
; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
-; SM60-NEXT: $L__BB191_5: // %atomicrmw.start24
+; SM60-NEXT: $L__BB191_5: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd8, %rd36;
-; SM60-NEXT: add.s64 %rd22, %rd8, 1;
-; SM60-NEXT: setp.ge.u64 %p5, %rd8, %rd2;
+; SM60-NEXT: mov.b64 %rd9, %rd36;
+; SM60-NEXT: add.s64 %rd22, %rd9, 1;
+; SM60-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
; SM60-NEXT: selp.b64 %rd23, 0, %rd22, %p5;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd8, %rd23;
-; SM60-NEXT: setp.ne.b64 %p6, %rd36, %rd8;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
; SM60-NEXT: @%p6 bra $L__BB191_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end23
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
-; SM60-NEXT: $L__BB191_7: // %atomicrmw.start32
+; SM60-NEXT: $L__BB191_7: // %atomicrmw.start26
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd9, %rd37;
-; SM60-NEXT: add.s64 %rd24, %rd9, 1;
-; SM60-NEXT: setp.ge.u64 %p7, %rd9, %rd7;
+; SM60-NEXT: mov.b64 %rd10, %rd37;
+; SM60-NEXT: add.s64 %rd24, %rd10, 1;
+; SM60-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
; SM60-NEXT: selp.b64 %rd25, 0, %rd24, %p7;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd9, %rd25;
-; SM60-NEXT: setp.ne.b64 %p8, %rd37, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
; SM60-NEXT: @%p8 bra $L__BB191_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end31
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
-; SM60-NEXT: $L__BB191_9: // %atomicrmw.start48
+; SM60-NEXT: $L__BB191_9: // %atomicrmw.start40
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd12, %rd38;
-; SM60-NEXT: add.s64 %rd26, %rd12, 1;
-; SM60-NEXT: setp.ge.u64 %p9, %rd12, %rd1;
+; SM60-NEXT: mov.b64 %rd13, %rd38;
+; SM60-NEXT: add.s64 %rd26, %rd13, 1;
+; SM60-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
; SM60-NEXT: selp.b64 %rd27, 0, %rd26, %p9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd12, %rd27;
-; SM60-NEXT: setp.ne.b64 %p10, %rd38, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM60-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
; SM60-NEXT: @%p10 bra $L__BB191_9;
-; SM60-NEXT: // %bb.10: // %atomicrmw.end47
+; SM60-NEXT: // %bb.10: // %atomicrmw.end39
; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
-; SM60-NEXT: $L__BB191_11: // %atomicrmw.start56
+; SM60-NEXT: $L__BB191_11: // %atomicrmw.start46
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd13, %rd39;
-; SM60-NEXT: add.s64 %rd28, %rd13, 1;
-; SM60-NEXT: setp.ge.u64 %p11, %rd13, %rd11;
+; SM60-NEXT: mov.b64 %rd14, %rd39;
+; SM60-NEXT: add.s64 %rd28, %rd14, 1;
+; SM60-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
; SM60-NEXT: selp.b64 %rd29, 0, %rd28, %p11;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd13, %rd29;
-; SM60-NEXT: setp.ne.b64 %p12, %rd39, %rd13;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM60-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
; SM60-NEXT: @%p12 bra $L__BB191_11;
-; SM60-NEXT: // %bb.12: // %atomicrmw.end55
+; SM60-NEXT: // %bb.12: // %atomicrmw.end45
; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
-; SM60-NEXT: $L__BB191_13: // %atomicrmw.start68
+; SM60-NEXT: $L__BB191_13: // %atomicrmw.start56
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd15, %rd40;
-; SM60-NEXT: add.s64 %rd30, %rd15, 1;
-; SM60-NEXT: setp.ge.u64 %p13, %rd15, %rd10;
+; SM60-NEXT: add.s64 %rd30, %rd40, 1;
+; SM60-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
; SM60-NEXT: selp.b64 %rd31, 0, %rd30, %p13;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd40, [%rd17+48], %rd15, %rd31;
-; SM60-NEXT: setp.ne.b64 %p14, %rd40, %rd15;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM60-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
+; SM60-NEXT: mov.b64 %rd40, %rd15;
; SM60-NEXT: @%p14 bra $L__BB191_13;
-; SM60-NEXT: // %bb.14: // %atomicrmw.end67
+; SM60-NEXT: // %bb.14: // %atomicrmw.end55
; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
-; SM60-NEXT: $L__BB191_15: // %atomicrmw.start76
+; SM60-NEXT: $L__BB191_15: // %atomicrmw.start62
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd32, %rd41, 1;
-; SM60-NEXT: setp.ge.u64 %p15, %rd41, %rd14;
+; SM60-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
; SM60-NEXT: selp.b64 %rd33, 0, %rd32, %p15;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM60-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
; SM60-NEXT: mov.b64 %rd41, %rd16;
; SM60-NEXT: @%p16 bra $L__BB191_15;
-; SM60-NEXT: // %bb.16: // %atomicrmw.end75
+; SM60-NEXT: // %bb.16: // %atomicrmw.end61
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd40, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
@@ -7324,9 +7342,9 @@ define <1 x i16> @udec_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i16_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NEXT: and.b32 %r5, %r4, 3;
@@ -7665,24 +7683,24 @@ define <1 x i64> @udec_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM60-NEXT: .reg .b64 %rd<8>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd7, [%rd3];
+; SM60-NEXT: ld.volatile.global.b64 %rd7, [%rd2];
; SM60-NEXT: $L__BB204_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd4, %rd7, -1;
; SM60-NEXT: setp.eq.b64 %p1, %rd7, 0;
-; SM60-NEXT: setp.gt.u64 %p2, %rd7, %rd1;
-; SM60-NEXT: selp.b64 %rd5, %rd1, %rd4, %p2;
-; SM60-NEXT: selp.b64 %rd6, %rd1, %rd5, %p1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd7, %rd6;
-; SM60-NEXT: setp.ne.b64 %p3, %rd2, %rd7;
-; SM60-NEXT: mov.b64 %rd7, %rd2;
+; SM60-NEXT: setp.gt.u64 %p2, %rd7, %rd3;
+; SM60-NEXT: selp.b64 %rd5, %rd3, %rd4, %p2;
+; SM60-NEXT: selp.b64 %rd6, %rd3, %rd5, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd1, [%rd2], %rd7, %rd6;
+; SM60-NEXT: setp.ne.b64 %p3, %rd1, %rd7;
+; SM60-NEXT: mov.b64 %rd7, %rd1;
; SM60-NEXT: @%p3 bra $L__BB204_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -7697,35 +7715,35 @@ define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd5, [udec_wrap_acq_rel_v2i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd5];
; SM60-NEXT: $L__BB205_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd3, %rd12;
-; SM60-NEXT: add.s64 %rd6, %rd3, -1;
-; SM60-NEXT: setp.eq.b64 %p1, %rd3, 0;
-; SM60-NEXT: setp.gt.u64 %p2, %rd3, %rd2;
-; SM60-NEXT: selp.b64 %rd7, %rd2, %rd6, %p2;
-; SM60-NEXT: selp.b64 %rd8, %rd2, %rd7, %p1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd5], %rd3, %rd8;
-; SM60-NEXT: setp.ne.b64 %p3, %rd12, %rd3;
+; SM60-NEXT: add.s64 %rd6, %rd12, -1;
+; SM60-NEXT: setp.eq.b64 %p1, %rd12, 0;
+; SM60-NEXT: setp.gt.u64 %p2, %rd12, %rd1;
+; SM60-NEXT: selp.b64 %rd7, %rd1, %rd6, %p2;
+; SM60-NEXT: selp.b64 %rd8, %rd1, %rd7, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd12, %rd8;
+; SM60-NEXT: setp.ne.b64 %p3, %rd3, %rd12;
+; SM60-NEXT: mov.b64 %rd12, %rd3;
; SM60-NEXT: @%p3 bra $L__BB205_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd5+8];
-; SM60-NEXT: $L__BB205_3: // %atomicrmw.start4
+; SM60-NEXT: $L__BB205_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd9, %rd13, -1;
; SM60-NEXT: setp.eq.b64 %p4, %rd13, 0;
-; SM60-NEXT: setp.gt.u64 %p5, %rd13, %rd1;
-; SM60-NEXT: selp.b64 %rd10, %rd1, %rd9, %p5;
-; SM60-NEXT: selp.b64 %rd11, %rd1, %rd10, %p4;
+; SM60-NEXT: setp.gt.u64 %p5, %rd13, %rd2;
+; SM60-NEXT: selp.b64 %rd10, %rd2, %rd9, %p5;
+; SM60-NEXT: selp.b64 %rd11, %rd2, %rd10, %p4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd13, %rd11;
; SM60-NEXT: setp.ne.b64 %p6, %rd4, %rd13;
; SM60-NEXT: mov.b64 %rd13, %rd4;
; SM60-NEXT: @%p6 bra $L__BB205_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end3
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd12, %rd4};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -7740,62 +7758,62 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd9, [udec_wrap_acq_rel_v4i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd9];
; SM60-NEXT: $L__BB206_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd4, %rd22;
-; SM60-NEXT: add.s64 %rd10, %rd4, -1;
-; SM60-NEXT: setp.eq.b64 %p1, %rd4, 0;
-; SM60-NEXT: setp.gt.u64 %p2, %rd4, %rd3;
+; SM60-NEXT: mov.b64 %rd5, %rd22;
+; SM60-NEXT: add.s64 %rd10, %rd5, -1;
+; SM60-NEXT: setp.eq.b64 %p1, %rd5, 0;
+; SM60-NEXT: setp.gt.u64 %p2, %rd5, %rd3;
; SM60-NEXT: selp.b64 %rd11, %rd3, %rd10, %p2;
; SM60-NEXT: selp.b64 %rd12, %rd3, %rd11, %p1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd22, [%rd9], %rd4, %rd12;
-; SM60-NEXT: setp.ne.b64 %p3, %rd22, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd22, [%rd9], %rd5, %rd12;
+; SM60-NEXT: setp.ne.b64 %p3, %rd22, %rd5;
; SM60-NEXT: @%p3 bra $L__BB206_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd23, [%rd9+8];
-; SM60-NEXT: $L__BB206_3: // %atomicrmw.start8
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd23;
-; SM60-NEXT: add.s64 %rd13, %rd5, -1;
-; SM60-NEXT: setp.eq.b64 %p4, %rd5, 0;
-; SM60-NEXT: setp.gt.u64 %p5, %rd5, %rd2;
-; SM60-NEXT: selp.b64 %rd14, %rd2, %rd13, %p5;
-; SM60-NEXT: selp.b64 %rd15, %rd2, %rd14, %p4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd9+8], %rd5, %rd15;
-; SM60-NEXT: setp.ne.b64 %p6, %rd23, %rd5;
+; SM60-NEXT: $L__BB206_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd23;
+; SM60-NEXT: add.s64 %rd13, %rd6, -1;
+; SM60-NEXT: setp.eq.b64 %p4, %rd6, 0;
+; SM60-NEXT: setp.gt.u64 %p5, %rd6, %rd4;
+; SM60-NEXT: selp.b64 %rd14, %rd4, %rd13, %p5;
+; SM60-NEXT: selp.b64 %rd15, %rd4, %rd14, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd9+8], %rd6, %rd15;
+; SM60-NEXT: setp.ne.b64 %p6, %rd23, %rd6;
; SM60-NEXT: @%p6 bra $L__BB206_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end7
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd9+16];
-; SM60-NEXT: $L__BB206_5: // %atomicrmw.start20
+; SM60-NEXT: $L__BB206_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd7, %rd24;
-; SM60-NEXT: add.s64 %rd16, %rd7, -1;
-; SM60-NEXT: setp.eq.b64 %p7, %rd7, 0;
-; SM60-NEXT: setp.gt.u64 %p8, %rd7, %rd1;
+; SM60-NEXT: add.s64 %rd16, %rd24, -1;
+; SM60-NEXT: setp.eq.b64 %p7, %rd24, 0;
+; SM60-NEXT: setp.gt.u64 %p8, %rd24, %rd1;
; SM60-NEXT: selp.b64 %rd17, %rd1, %rd16, %p8;
; SM60-NEXT: selp.b64 %rd18, %rd1, %rd17, %p7;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd24, [%rd9+16], %rd7, %rd18;
-; SM60-NEXT: setp.ne.b64 %p9, %rd24, %rd7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd24, %rd18;
+; SM60-NEXT: setp.ne.b64 %p9, %rd7, %rd24;
+; SM60-NEXT: mov.b64 %rd24, %rd7;
; SM60-NEXT: @%p9 bra $L__BB206_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd25, [%rd9+24];
-; SM60-NEXT: $L__BB206_7: // %atomicrmw.start28
+; SM60-NEXT: $L__BB206_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd19, %rd25, -1;
; SM60-NEXT: setp.eq.b64 %p10, %rd25, 0;
-; SM60-NEXT: setp.gt.u64 %p11, %rd25, %rd6;
-; SM60-NEXT: selp.b64 %rd20, %rd6, %rd19, %p11;
-; SM60-NEXT: selp.b64 %rd21, %rd6, %rd20, %p10;
+; SM60-NEXT: setp.gt.u64 %p11, %rd25, %rd2;
+; SM60-NEXT: selp.b64 %rd20, %rd2, %rd19, %p11;
+; SM60-NEXT: selp.b64 %rd21, %rd2, %rd20, %p10;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd25, %rd21;
; SM60-NEXT: setp.ne.b64 %p12, %rd8, %rd25;
; SM60-NEXT: mov.b64 %rd25, %rd8;
; SM60-NEXT: @%p12 bra $L__BB206_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end27
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd24, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd22, %rd23};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
@@ -7811,116 +7829,116 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd17, [udec_wrap_acq_rel_v8i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+48];
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+32];
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd42, [%rd17];
; SM60-NEXT: $L__BB207_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd42;
-; SM60-NEXT: add.s64 %rd18, %rd5, -1;
-; SM60-NEXT: setp.eq.b64 %p1, %rd5, 0;
-; SM60-NEXT: setp.gt.u64 %p2, %rd5, %rd4;
-; SM60-NEXT: selp.b64 %rd19, %rd4, %rd18, %p2;
-; SM60-NEXT: selp.b64 %rd20, %rd4, %rd19, %p1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd42, [%rd17], %rd5, %rd20;
-; SM60-NEXT: setp.ne.b64 %p3, %rd42, %rd5;
+; SM60-NEXT: mov.b64 %rd7, %rd42;
+; SM60-NEXT: add.s64 %rd18, %rd7, -1;
+; SM60-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM60-NEXT: setp.gt.u64 %p2, %rd7, %rd5;
+; SM60-NEXT: selp.b64 %rd19, %rd5, %rd18, %p2;
+; SM60-NEXT: selp.b64 %rd20, %rd5, %rd19, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd42, [%rd17], %rd7, %rd20;
+; SM60-NEXT: setp.ne.b64 %p3, %rd42, %rd7;
; SM60-NEXT: @%p3 bra $L__BB207_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd43, [%rd17+8];
-; SM60-NEXT: $L__BB207_3: // %atomicrmw.start12
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd6, %rd43;
-; SM60-NEXT: add.s64 %rd21, %rd6, -1;
-; SM60-NEXT: setp.eq.b64 %p4, %rd6, 0;
-; SM60-NEXT: setp.gt.u64 %p5, %rd6, %rd3;
-; SM60-NEXT: selp.b64 %rd22, %rd3, %rd21, %p5;
-; SM60-NEXT: selp.b64 %rd23, %rd3, %rd22, %p4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd43, [%rd17+8], %rd6, %rd23;
-; SM60-NEXT: setp.ne.b64 %p6, %rd43, %rd6;
+; SM60-NEXT: $L__BB207_3: // %atomicrmw.start10
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd43;
+; SM60-NEXT: add.s64 %rd21, %rd8, -1;
+; SM60-NEXT: setp.eq.b64 %p4, %rd8, 0;
+; SM60-NEXT: setp.gt.u64 %p5, %rd8, %rd6;
+; SM60-NEXT: selp.b64 %rd22, %rd6, %rd21, %p5;
+; SM60-NEXT: selp.b64 %rd23, %rd6, %rd22, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd43, [%rd17+8], %rd8, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd43, %rd8;
; SM60-NEXT: @%p6 bra $L__BB207_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end11
+; SM60-NEXT: // %bb.4: // %atomicrmw.end9
; SM60-NEXT: ld.volatile.global.b64 %rd44, [%rd17+16];
-; SM60-NEXT: $L__BB207_5: // %atomicrmw.start24
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd8, %rd44;
-; SM60-NEXT: add.s64 %rd24, %rd8, -1;
-; SM60-NEXT: setp.eq.b64 %p7, %rd8, 0;
-; SM60-NEXT: setp.gt.u64 %p8, %rd8, %rd2;
-; SM60-NEXT: selp.b64 %rd25, %rd2, %rd24, %p8;
-; SM60-NEXT: selp.b64 %rd26, %rd2, %rd25, %p7;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd44, [%rd17+16], %rd8, %rd26;
-; SM60-NEXT: setp.ne.b64 %p9, %rd44, %rd8;
+; SM60-NEXT: $L__BB207_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd44;
+; SM60-NEXT: add.s64 %rd24, %rd9, -1;
+; SM60-NEXT: setp.eq.b64 %p7, %rd9, 0;
+; SM60-NEXT: setp.gt.u64 %p8, %rd9, %rd3;
+; SM60-NEXT: selp.b64 %rd25, %rd3, %rd24, %p8;
+; SM60-NEXT: selp.b64 %rd26, %rd3, %rd25, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd44, [%rd17+16], %rd9, %rd26;
+; SM60-NEXT: setp.ne.b64 %p9, %rd44, %rd9;
; SM60-NEXT: @%p9 bra $L__BB207_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end23
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
; SM60-NEXT: ld.volatile.global.b64 %rd45, [%rd17+24];
-; SM60-NEXT: $L__BB207_7: // %atomicrmw.start32
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd9, %rd45;
-; SM60-NEXT: add.s64 %rd27, %rd9, -1;
-; SM60-NEXT: setp.eq.b64 %p10, %rd9, 0;
-; SM60-NEXT: setp.gt.u64 %p11, %rd9, %rd7;
-; SM60-NEXT: selp.b64 %rd28, %rd7, %rd27, %p11;
-; SM60-NEXT: selp.b64 %rd29, %rd7, %rd28, %p10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd17+24], %rd9, %rd29;
-; SM60-NEXT: setp.ne.b64 %p12, %rd45, %rd9;
+; SM60-NEXT: $L__BB207_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd45;
+; SM60-NEXT: add.s64 %rd27, %rd10, -1;
+; SM60-NEXT: setp.eq.b64 %p10, %rd10, 0;
+; SM60-NEXT: setp.gt.u64 %p11, %rd10, %rd4;
+; SM60-NEXT: selp.b64 %rd28, %rd4, %rd27, %p11;
+; SM60-NEXT: selp.b64 %rd29, %rd4, %rd28, %p10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd17+24], %rd10, %rd29;
+; SM60-NEXT: setp.ne.b64 %p12, %rd45, %rd10;
; SM60-NEXT: @%p12 bra $L__BB207_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end31
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
; SM60-NEXT: ld.volatile.global.b64 %rd46, [%rd17+32];
-; SM60-NEXT: $L__BB207_9: // %atomicrmw.start48
+; SM60-NEXT: $L__BB207_9: // %atomicrmw.start40
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd12, %rd46;
-; SM60-NEXT: add.s64 %rd30, %rd12, -1;
-; SM60-NEXT: setp.eq.b64 %p13, %rd12, 0;
-; SM60-NEXT: setp.gt.u64 %p14, %rd12, %rd1;
+; SM60-NEXT: mov.b64 %rd13, %rd46;
+; SM60-NEXT: add.s64 %rd30, %rd13, -1;
+; SM60-NEXT: setp.eq.b64 %p13, %rd13, 0;
+; SM60-NEXT: setp.gt.u64 %p14, %rd13, %rd1;
; SM60-NEXT: selp.b64 %rd31, %rd1, %rd30, %p14;
; SM60-NEXT: selp.b64 %rd32, %rd1, %rd31, %p13;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd46, [%rd17+32], %rd12, %rd32;
-; SM60-NEXT: setp.ne.b64 %p15, %rd46, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd46, [%rd17+32], %rd13, %rd32;
+; SM60-NEXT: setp.ne.b64 %p15, %rd46, %rd13;
; SM60-NEXT: @%p15 bra $L__BB207_9;
-; SM60-NEXT: // %bb.10: // %atomicrmw.end47
+; SM60-NEXT: // %bb.10: // %atomicrmw.end39
; SM60-NEXT: ld.volatile.global.b64 %rd47, [%rd17+40];
-; SM60-NEXT: $L__BB207_11: // %atomicrmw.start56
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd13, %rd47;
-; SM60-NEXT: add.s64 %rd33, %rd13, -1;
-; SM60-NEXT: setp.eq.b64 %p16, %rd13, 0;
-; SM60-NEXT: setp.gt.u64 %p17, %rd13, %rd11;
-; SM60-NEXT: selp.b64 %rd34, %rd11, %rd33, %p17;
-; SM60-NEXT: selp.b64 %rd35, %rd11, %rd34, %p16;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd47, [%rd17+40], %rd13, %rd35;
-; SM60-NEXT: setp.ne.b64 %p18, %rd47, %rd13;
+; SM60-NEXT: $L__BB207_11: // %atomicrmw.start46
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd47;
+; SM60-NEXT: add.s64 %rd33, %rd14, -1;
+; SM60-NEXT: setp.eq.b64 %p16, %rd14, 0;
+; SM60-NEXT: setp.gt.u64 %p17, %rd14, %rd2;
+; SM60-NEXT: selp.b64 %rd34, %rd2, %rd33, %p17;
+; SM60-NEXT: selp.b64 %rd35, %rd2, %rd34, %p16;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd47, [%rd17+40], %rd14, %rd35;
+; SM60-NEXT: setp.ne.b64 %p18, %rd47, %rd14;
; SM60-NEXT: @%p18 bra $L__BB207_11;
-; SM60-NEXT: // %bb.12: // %atomicrmw.end55
+; SM60-NEXT: // %bb.12: // %atomicrmw.end45
; SM60-NEXT: ld.volatile.global.b64 %rd48, [%rd17+48];
-; SM60-NEXT: $L__BB207_13: // %atomicrmw.start68
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd15, %rd48;
-; SM60-NEXT: add.s64 %rd36, %rd15, -1;
-; SM60-NEXT: setp.eq.b64 %p19, %rd15, 0;
-; SM60-NEXT: setp.gt.u64 %p20, %rd15, %rd10;
-; SM60-NEXT: selp.b64 %rd37, %rd10, %rd36, %p20;
-; SM60-NEXT: selp.b64 %rd38, %rd10, %rd37, %p19;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd48, [%rd17+48], %rd15, %rd38;
-; SM60-NEXT: setp.ne.b64 %p21, %rd48, %rd15;
+; SM60-NEXT: $L__BB207_13: // %atomicrmw.start56
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd36, %rd48, -1;
+; SM60-NEXT: setp.eq.b64 %p19, %rd48, 0;
+; SM60-NEXT: setp.gt.u64 %p20, %rd48, %rd11;
+; SM60-NEXT: selp.b64 %rd37, %rd11, %rd36, %p20;
+; SM60-NEXT: selp.b64 %rd38, %rd11, %rd37, %p19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd48, %rd38;
+; SM60-NEXT: setp.ne.b64 %p21, %rd15, %rd48;
+; SM60-NEXT: mov.b64 %rd48, %rd15;
; SM60-NEXT: @%p21 bra $L__BB207_13;
-; SM60-NEXT: // %bb.14: // %atomicrmw.end67
+; SM60-NEXT: // %bb.14: // %atomicrmw.end55
; SM60-NEXT: ld.volatile.global.b64 %rd49, [%rd17+56];
-; SM60-NEXT: $L__BB207_15: // %atomicrmw.start76
+; SM60-NEXT: $L__BB207_15: // %atomicrmw.start62
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd39, %rd49, -1;
; SM60-NEXT: setp.eq.b64 %p22, %rd49, 0;
-; SM60-NEXT: setp.gt.u64 %p23, %rd49, %rd14;
-; SM60-NEXT: selp.b64 %rd40, %rd14, %rd39, %p23;
-; SM60-NEXT: selp.b64 %rd41, %rd14, %rd40, %p22;
+; SM60-NEXT: setp.gt.u64 %p23, %rd49, %rd12;
+; SM60-NEXT: selp.b64 %rd40, %rd12, %rd39, %p23;
+; SM60-NEXT: selp.b64 %rd41, %rd12, %rd40, %p22;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd49, %rd41;
; SM60-NEXT: setp.ne.b64 %p24, %rd16, %rd49;
; SM60-NEXT: mov.b64 %rd49, %rd16;
; SM60-NEXT: @%p24 bra $L__BB207_15;
-; SM60-NEXT: // %bb.16: // %atomicrmw.end75
+; SM60-NEXT: // %bb.16: // %atomicrmw.end61
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd48, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd46, %rd47};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
@@ -8206,9 +8224,9 @@ define <1 x i16> @usub_cond_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i16_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NEXT: and.b32 %r5, %r4, 3;
@@ -8433,22 +8451,22 @@ define <1 x i32> @usub_cond_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v1i32_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i32_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b32 %r1, [usub_cond_acq_rel_v1i32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
; SM60-NEXT: $L__BB216_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.ge.u32 %p1, %r5, %r1;
-; SM60-NEXT: sub.s32 %r3, %r5, %r1;
+; SM60-NEXT: setp.ge.u32 %p1, %r5, %r2;
+; SM60-NEXT: sub.s32 %r3, %r5, %r2;
; SM60-NEXT: selp.b32 %r4, %r3, %r5, %p1;
-; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM60-NEXT: setp.ne.b32 %p2, %r2, %r5;
-; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p2, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
; SM60-NEXT: @%p2 bra $L__BB216_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -8692,22 +8710,22 @@ define <1 x i64> @usub_cond_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM60-NEXT: .reg .b64 %rd<7>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd3];
+; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd2];
; SM60-NEXT: $L__BB220_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
-; SM60-NEXT: sub.s64 %rd4, %rd6, %rd1;
+; SM60-NEXT: setp.ge.u64 %p1, %rd6, %rd3;
+; SM60-NEXT: sub.s64 %rd4, %rd6, %rd3;
; SM60-NEXT: selp.b64 %rd5, %rd4, %rd6, %p1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
-; SM60-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
-; SM60-NEXT: mov.b64 %rd6, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5;
+; SM60-NEXT: setp.ne.b64 %p2, %rd1, %rd6;
+; SM60-NEXT: mov.b64 %rd6, %rd1;
; SM60-NEXT: @%p2 bra $L__BB220_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -8722,31 +8740,31 @@ define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd5, [usub_cond_acq_rel_v2i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
; SM60-NEXT: $L__BB221_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd3, %rd10;
-; SM60-NEXT: setp.ge.u64 %p1, %rd3, %rd2;
-; SM60-NEXT: sub.s64 %rd6, %rd3, %rd2;
-; SM60-NEXT: selp.b64 %rd7, %rd6, %rd3, %p1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd10, [%rd5], %rd3, %rd7;
-; SM60-NEXT: setp.ne.b64 %p2, %rd10, %rd3;
+; SM60-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
+; SM60-NEXT: sub.s64 %rd6, %rd10, %rd1;
+; SM60-NEXT: selp.b64 %rd7, %rd6, %rd10, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM60-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
+; SM60-NEXT: mov.b64 %rd10, %rd3;
; SM60-NEXT: @%p2 bra $L__BB221_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
-; SM60-NEXT: $L__BB221_3: // %atomicrmw.start4
+; SM60-NEXT: $L__BB221_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.ge.u64 %p3, %rd11, %rd1;
-; SM60-NEXT: sub.s64 %rd8, %rd11, %rd1;
+; SM60-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
+; SM60-NEXT: sub.s64 %rd8, %rd11, %rd2;
; SM60-NEXT: selp.b64 %rd9, %rd8, %rd11, %p3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM60-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
; SM60-NEXT: mov.b64 %rd11, %rd4;
; SM60-NEXT: @%p4 bra $L__BB221_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end3
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd4};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -8761,54 +8779,54 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd9, [usub_cond_acq_rel_v4i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
; SM60-NEXT: $L__BB222_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd4, %rd18;
-; SM60-NEXT: setp.ge.u64 %p1, %rd4, %rd3;
-; SM60-NEXT: sub.s64 %rd10, %rd4, %rd3;
-; SM60-NEXT: selp.b64 %rd11, %rd10, %rd4, %p1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd4, %rd11;
-; SM60-NEXT: setp.ne.b64 %p2, %rd18, %rd4;
+; SM60-NEXT: mov.b64 %rd5, %rd18;
+; SM60-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
+; SM60-NEXT: sub.s64 %rd10, %rd5, %rd3;
+; SM60-NEXT: selp.b64 %rd11, %rd10, %rd5, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
; SM60-NEXT: @%p2 bra $L__BB222_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
-; SM60-NEXT: $L__BB222_3: // %atomicrmw.start8
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd19;
-; SM60-NEXT: setp.ge.u64 %p3, %rd5, %rd2;
-; SM60-NEXT: sub.s64 %rd12, %rd5, %rd2;
-; SM60-NEXT: selp.b64 %rd13, %rd12, %rd5, %p3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd5, %rd13;
-; SM60-NEXT: setp.ne.b64 %p4, %rd19, %rd5;
+; SM60-NEXT: $L__BB222_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd19;
+; SM60-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
+; SM60-NEXT: sub.s64 %rd12, %rd6, %rd4;
+; SM60-NEXT: selp.b64 %rd13, %rd12, %rd6, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
; SM60-NEXT: @%p4 bra $L__BB222_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end7
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
-; SM60-NEXT: $L__BB222_5: // %atomicrmw.start20
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd7, %rd20;
-; SM60-NEXT: setp.ge.u64 %p5, %rd7, %rd1;
-; SM60-NEXT: sub.s64 %rd14, %rd7, %rd1;
-; SM60-NEXT: selp.b64 %rd15, %rd14, %rd7, %p5;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd20, [%rd9+16], %rd7, %rd15;
-; SM60-NEXT: setp.ne.b64 %p6, %rd20, %rd7;
+; SM60-NEXT: $L__BB222_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
+; SM60-NEXT: sub.s64 %rd14, %rd20, %rd1;
+; SM60-NEXT: selp.b64 %rd15, %rd14, %rd20, %p5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM60-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
+; SM60-NEXT: mov.b64 %rd20, %rd7;
; SM60-NEXT: @%p6 bra $L__BB222_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
-; SM60-NEXT: $L__BB222_7: // %atomicrmw.start28
+; SM60-NEXT: $L__BB222_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.ge.u64 %p7, %rd21, %rd6;
-; SM60-NEXT: sub.s64 %rd16, %rd21, %rd6;
+; SM60-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
+; SM60-NEXT: sub.s64 %rd16, %rd21, %rd2;
; SM60-NEXT: selp.b64 %rd17, %rd16, %rd21, %p7;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM60-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
; SM60-NEXT: mov.b64 %rd21, %rd8;
; SM60-NEXT: @%p8 bra $L__BB222_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end27
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd20, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
@@ -8824,100 +8842,100 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd17, [usub_cond_acq_rel_v8i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [usub_cond_acq_rel_v8i64_global_cta_param_1+48];
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [usub_cond_acq_rel_v8i64_global_cta_param_1+32];
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [usub_cond_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
; SM60-NEXT: $L__BB223_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd34;
-; SM60-NEXT: setp.ge.u64 %p1, %rd5, %rd4;
-; SM60-NEXT: sub.s64 %rd18, %rd5, %rd4;
-; SM60-NEXT: selp.b64 %rd19, %rd18, %rd5, %p1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd5, %rd19;
-; SM60-NEXT: setp.ne.b64 %p2, %rd34, %rd5;
+; SM60-NEXT: mov.b64 %rd7, %rd34;
+; SM60-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
+; SM60-NEXT: sub.s64 %rd18, %rd7, %rd5;
+; SM60-NEXT: selp.b64 %rd19, %rd18, %rd7, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
; SM60-NEXT: @%p2 bra $L__BB223_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
-; SM60-NEXT: $L__BB223_3: // %atomicrmw.start12
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd6, %rd35;
-; SM60-NEXT: setp.ge.u64 %p3, %rd6, %rd3;
-; SM60-NEXT: sub.s64 %rd20, %rd6, %rd3;
-; SM60-NEXT: selp.b64 %rd21, %rd20, %rd6, %p3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd6, %rd21;
-; SM60-NEXT: setp.ne.b64 %p4, %rd35, %rd6;
+; SM60-NEXT: $L__BB223_3: // %atomicrmw.start10
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd35;
+; SM60-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
+; SM60-NEXT: sub.s64 %rd20, %rd8, %rd6;
+; SM60-NEXT: selp.b64 %rd21, %rd20, %rd8, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
; SM60-NEXT: @%p4 bra $L__BB223_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end11
+; SM60-NEXT: // %bb.4: // %atomicrmw.end9
; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
-; SM60-NEXT: $L__BB223_5: // %atomicrmw.start24
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd8, %rd36;
-; SM60-NEXT: setp.ge.u64 %p5, %rd8, %rd2;
-; SM60-NEXT: sub.s64 %rd22, %rd8, %rd2;
-; SM60-NEXT: selp.b64 %rd23, %rd22, %rd8, %p5;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd8, %rd23;
-; SM60-NEXT: setp.ne.b64 %p6, %rd36, %rd8;
+; SM60-NEXT: $L__BB223_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd36;
+; SM60-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
+; SM60-NEXT: sub.s64 %rd22, %rd9, %rd3;
+; SM60-NEXT: selp.b64 %rd23, %rd22, %rd9, %p5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
; SM60-NEXT: @%p6 bra $L__BB223_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end23
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
-; SM60-NEXT: $L__BB223_7: // %atomicrmw.start32
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd9, %rd37;
-; SM60-NEXT: setp.ge.u64 %p7, %rd9, %rd7;
-; SM60-NEXT: sub.s64 %rd24, %rd9, %rd7;
-; SM60-NEXT: selp.b64 %rd25, %rd24, %rd9, %p7;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd9, %rd25;
-; SM60-NEXT: setp.ne.b64 %p8, %rd37, %rd9;
+; SM60-NEXT: $L__BB223_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd37;
+; SM60-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
+; SM60-NEXT: sub.s64 %rd24, %rd10, %rd4;
+; SM60-NEXT: selp.b64 %rd25, %rd24, %rd10, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
; SM60-NEXT: @%p8 bra $L__BB223_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end31
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
-; SM60-NEXT: $L__BB223_9: // %atomicrmw.start48
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd12, %rd38;
-; SM60-NEXT: setp.ge.u64 %p9, %rd12, %rd1;
-; SM60-NEXT: sub.s64 %rd26, %rd12, %rd1;
-; SM60-NEXT: selp.b64 %rd27, %rd26, %rd12, %p9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd12, %rd27;
-; SM60-NEXT: setp.ne.b64 %p10, %rd38, %rd12;
+; SM60-NEXT: $L__BB223_9: // %atomicrmw.start40
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd38;
+; SM60-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
+; SM60-NEXT: sub.s64 %rd26, %rd13, %rd1;
+; SM60-NEXT: selp.b64 %rd27, %rd26, %rd13, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM60-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
; SM60-NEXT: @%p10 bra $L__BB223_9;
-; SM60-NEXT: // %bb.10: // %atomicrmw.end47
+; SM60-NEXT: // %bb.10: // %atomicrmw.end39
; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
-; SM60-NEXT: $L__BB223_11: // %atomicrmw.start56
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd13, %rd39;
-; SM60-NEXT: setp.ge.u64 %p11, %rd13, %rd11;
-; SM60-NEXT: sub.s64 %rd28, %rd13, %rd11;
-; SM60-NEXT: selp.b64 %rd29, %rd28, %rd13, %p11;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd13, %rd29;
-; SM60-NEXT: setp.ne.b64 %p12, %rd39, %rd13;
+; SM60-NEXT: $L__BB223_11: // %atomicrmw.start46
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd39;
+; SM60-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
+; SM60-NEXT: sub.s64 %rd28, %rd14, %rd2;
+; SM60-NEXT: selp.b64 %rd29, %rd28, %rd14, %p11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM60-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
; SM60-NEXT: @%p12 bra $L__BB223_11;
-; SM60-NEXT: // %bb.12: // %atomicrmw.end55
+; SM60-NEXT: // %bb.12: // %atomicrmw.end45
; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
-; SM60-NEXT: $L__BB223_13: // %atomicrmw.start68
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd15, %rd40;
-; SM60-NEXT: setp.ge.u64 %p13, %rd15, %rd10;
-; SM60-NEXT: sub.s64 %rd30, %rd15, %rd10;
-; SM60-NEXT: selp.b64 %rd31, %rd30, %rd15, %p13;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd40, [%rd17+48], %rd15, %rd31;
-; SM60-NEXT: setp.ne.b64 %p14, %rd40, %rd15;
+; SM60-NEXT: $L__BB223_13: // %atomicrmw.start56
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
+; SM60-NEXT: sub.s64 %rd30, %rd40, %rd11;
+; SM60-NEXT: selp.b64 %rd31, %rd30, %rd40, %p13;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM60-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
+; SM60-NEXT: mov.b64 %rd40, %rd15;
; SM60-NEXT: @%p14 bra $L__BB223_13;
-; SM60-NEXT: // %bb.14: // %atomicrmw.end67
+; SM60-NEXT: // %bb.14: // %atomicrmw.end55
; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
-; SM60-NEXT: $L__BB223_15: // %atomicrmw.start76
+; SM60-NEXT: $L__BB223_15: // %atomicrmw.start62
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.ge.u64 %p15, %rd41, %rd14;
-; SM60-NEXT: sub.s64 %rd32, %rd41, %rd14;
+; SM60-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
+; SM60-NEXT: sub.s64 %rd32, %rd41, %rd12;
; SM60-NEXT: selp.b64 %rd33, %rd32, %rd41, %p15;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM60-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
; SM60-NEXT: mov.b64 %rd41, %rd16;
; SM60-NEXT: @%p16 bra $L__BB223_15;
-; SM60-NEXT: // %bb.16: // %atomicrmw.end75
+; SM60-NEXT: // %bb.16: // %atomicrmw.end61
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd40, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
@@ -9195,9 +9213,9 @@ define <1 x i16> @usub_sat_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i16_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NEXT: and.b32 %r5, %r4, 3;
@@ -9442,21 +9460,21 @@ define <1 x i32> @usub_sat_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v1i32_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i32_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b32 %r1, [usub_sat_acq_rel_v1i32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
; SM60-NEXT: $L__BB232_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: max.u32 %r3, %r5, %r1;
-; SM60-NEXT: sub.s32 %r4, %r3, %r1;
-; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: max.u32 %r3, %r5, %r2;
+; SM60-NEXT: sub.s32 %r4, %r3, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
; SM60-NEXT: @%p1 bra $L__BB232_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -9686,21 +9704,21 @@ define <1 x i64> @usub_sat_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-NEXT: .reg .b64 %rd<7>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd3];
+; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd2];
; SM60-NEXT: $L__BB236_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: max.u64 %rd4, %rd6, %rd1;
-; SM60-NEXT: sub.s64 %rd5, %rd4, %rd1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
-; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
-; SM60-NEXT: mov.b64 %rd6, %rd2;
+; SM60-NEXT: max.u64 %rd4, %rd6, %rd3;
+; SM60-NEXT: sub.s64 %rd5, %rd4, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5;
+; SM60-NEXT: setp.ne.b64 %p1, %rd1, %rd6;
+; SM60-NEXT: mov.b64 %rd6, %rd1;
; SM60-NEXT: @%p1 bra $L__BB236_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -9715,29 +9733,29 @@ define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd5, [usub_sat_acq_rel_v2i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
; SM60-NEXT: $L__BB237_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd3, %rd10;
-; SM60-NEXT: max.u64 %rd6, %rd3, %rd2;
-; SM60-NEXT: sub.s64 %rd7, %rd6, %rd2;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd10, [%rd5], %rd3, %rd7;
-; SM60-NEXT: setp.ne.b64 %p1, %rd10, %rd3;
+; SM60-NEXT: max.u64 %rd6, %rd10, %rd1;
+; SM60-NEXT: sub.s64 %rd7, %rd6, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
+; SM60-NEXT: mov.b64 %rd10, %rd3;
; SM60-NEXT: @%p1 bra $L__BB237_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
-; SM60-NEXT: $L__BB237_3: // %atomicrmw.start4
+; SM60-NEXT: $L__BB237_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: max.u64 %rd8, %rd11, %rd1;
-; SM60-NEXT: sub.s64 %rd9, %rd8, %rd1;
+; SM60-NEXT: max.u64 %rd8, %rd11, %rd2;
+; SM60-NEXT: sub.s64 %rd9, %rd8, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
; SM60-NEXT: mov.b64 %rd11, %rd4;
; SM60-NEXT: @%p2 bra $L__BB237_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end3
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd4};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -9752,50 +9770,50 @@ define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd9, [usub_sat_acq_rel_v4i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
; SM60-NEXT: $L__BB238_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd4, %rd18;
-; SM60-NEXT: max.u64 %rd10, %rd4, %rd3;
+; SM60-NEXT: mov.b64 %rd5, %rd18;
+; SM60-NEXT: max.u64 %rd10, %rd5, %rd3;
; SM60-NEXT: sub.s64 %rd11, %rd10, %rd3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd4, %rd11;
-; SM60-NEXT: setp.ne.b64 %p1, %rd18, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
; SM60-NEXT: @%p1 bra $L__BB238_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
-; SM60-NEXT: $L__BB238_3: // %atomicrmw.start8
+; SM60-NEXT: $L__BB238_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd19;
-; SM60-NEXT: max.u64 %rd12, %rd5, %rd2;
-; SM60-NEXT: sub.s64 %rd13, %rd12, %rd2;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd5, %rd13;
-; SM60-NEXT: setp.ne.b64 %p2, %rd19, %rd5;
+; SM60-NEXT: mov.b64 %rd6, %rd19;
+; SM60-NEXT: max.u64 %rd12, %rd6, %rd4;
+; SM60-NEXT: sub.s64 %rd13, %rd12, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM60-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
; SM60-NEXT: @%p2 bra $L__BB238_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end7
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
-; SM60-NEXT: $L__BB238_5: // %atomicrmw.start20
+; SM60-NEXT: $L__BB238_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd7, %rd20;
-; SM60-NEXT: max.u64 %rd14, %rd7, %rd1;
+; SM60-NEXT: max.u64 %rd14, %rd20, %rd1;
; SM60-NEXT: sub.s64 %rd15, %rd14, %rd1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd20, [%rd9+16], %rd7, %rd15;
-; SM60-NEXT: setp.ne.b64 %p3, %rd20, %rd7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
+; SM60-NEXT: mov.b64 %rd20, %rd7;
; SM60-NEXT: @%p3 bra $L__BB238_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
-; SM60-NEXT: $L__BB238_7: // %atomicrmw.start28
+; SM60-NEXT: $L__BB238_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: max.u64 %rd16, %rd21, %rd6;
-; SM60-NEXT: sub.s64 %rd17, %rd16, %rd6;
+; SM60-NEXT: max.u64 %rd16, %rd21, %rd2;
+; SM60-NEXT: sub.s64 %rd17, %rd16, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
; SM60-NEXT: mov.b64 %rd21, %rd8;
; SM60-NEXT: @%p4 bra $L__BB238_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end27
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd20, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
@@ -9811,92 +9829,92 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd17, [usub_sat_acq_rel_v8i64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [usub_sat_acq_rel_v8i64_global_cta_param_1+48];
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [usub_sat_acq_rel_v8i64_global_cta_param_1+32];
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [usub_sat_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
; SM60-NEXT: $L__BB239_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd34;
-; SM60-NEXT: max.u64 %rd18, %rd5, %rd4;
-; SM60-NEXT: sub.s64 %rd19, %rd18, %rd4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd5, %rd19;
-; SM60-NEXT: setp.ne.b64 %p1, %rd34, %rd5;
+; SM60-NEXT: mov.b64 %rd7, %rd34;
+; SM60-NEXT: max.u64 %rd18, %rd7, %rd5;
+; SM60-NEXT: sub.s64 %rd19, %rd18, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM60-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
; SM60-NEXT: @%p1 bra $L__BB239_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
-; SM60-NEXT: $L__BB239_3: // %atomicrmw.start12
+; SM60-NEXT: $L__BB239_3: // %atomicrmw.start10
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd6, %rd35;
-; SM60-NEXT: max.u64 %rd20, %rd6, %rd3;
-; SM60-NEXT: sub.s64 %rd21, %rd20, %rd3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd6, %rd21;
-; SM60-NEXT: setp.ne.b64 %p2, %rd35, %rd6;
+; SM60-NEXT: mov.b64 %rd8, %rd35;
+; SM60-NEXT: max.u64 %rd20, %rd8, %rd6;
+; SM60-NEXT: sub.s64 %rd21, %rd20, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM60-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
; SM60-NEXT: @%p2 bra $L__BB239_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end11
+; SM60-NEXT: // %bb.4: // %atomicrmw.end9
; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
-; SM60-NEXT: $L__BB239_5: // %atomicrmw.start24
+; SM60-NEXT: $L__BB239_5: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd8, %rd36;
-; SM60-NEXT: max.u64 %rd22, %rd8, %rd2;
-; SM60-NEXT: sub.s64 %rd23, %rd22, %rd2;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd8, %rd23;
-; SM60-NEXT: setp.ne.b64 %p3, %rd36, %rd8;
+; SM60-NEXT: mov.b64 %rd9, %rd36;
+; SM60-NEXT: max.u64 %rd22, %rd9, %rd3;
+; SM60-NEXT: sub.s64 %rd23, %rd22, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM60-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
; SM60-NEXT: @%p3 bra $L__BB239_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end23
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
-; SM60-NEXT: $L__BB239_7: // %atomicrmw.start32
+; SM60-NEXT: $L__BB239_7: // %atomicrmw.start26
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd9, %rd37;
-; SM60-NEXT: max.u64 %rd24, %rd9, %rd7;
-; SM60-NEXT: sub.s64 %rd25, %rd24, %rd7;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd9, %rd25;
-; SM60-NEXT: setp.ne.b64 %p4, %rd37, %rd9;
+; SM60-NEXT: mov.b64 %rd10, %rd37;
+; SM60-NEXT: max.u64 %rd24, %rd10, %rd4;
+; SM60-NEXT: sub.s64 %rd25, %rd24, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM60-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
; SM60-NEXT: @%p4 bra $L__BB239_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end31
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
-; SM60-NEXT: $L__BB239_9: // %atomicrmw.start48
+; SM60-NEXT: $L__BB239_9: // %atomicrmw.start40
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd12, %rd38;
-; SM60-NEXT: max.u64 %rd26, %rd12, %rd1;
+; SM60-NEXT: mov.b64 %rd13, %rd38;
+; SM60-NEXT: max.u64 %rd26, %rd13, %rd1;
; SM60-NEXT: sub.s64 %rd27, %rd26, %rd1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd12, %rd27;
-; SM60-NEXT: setp.ne.b64 %p5, %rd38, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM60-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
; SM60-NEXT: @%p5 bra $L__BB239_9;
-; SM60-NEXT: // %bb.10: // %atomicrmw.end47
+; SM60-NEXT: // %bb.10: // %atomicrmw.end39
; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
-; SM60-NEXT: $L__BB239_11: // %atomicrmw.start56
+; SM60-NEXT: $L__BB239_11: // %atomicrmw.start46
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd13, %rd39;
-; SM60-NEXT: max.u64 %rd28, %rd13, %rd11;
-; SM60-NEXT: sub.s64 %rd29, %rd28, %rd11;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd13, %rd29;
-; SM60-NEXT: setp.ne.b64 %p6, %rd39, %rd13;
+; SM60-NEXT: mov.b64 %rd14, %rd39;
+; SM60-NEXT: max.u64 %rd28, %rd14, %rd2;
+; SM60-NEXT: sub.s64 %rd29, %rd28, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM60-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
; SM60-NEXT: @%p6 bra $L__BB239_11;
-; SM60-NEXT: // %bb.12: // %atomicrmw.end55
+; SM60-NEXT: // %bb.12: // %atomicrmw.end45
; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
-; SM60-NEXT: $L__BB239_13: // %atomicrmw.start68
+; SM60-NEXT: $L__BB239_13: // %atomicrmw.start56
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd15, %rd40;
-; SM60-NEXT: max.u64 %rd30, %rd15, %rd10;
-; SM60-NEXT: sub.s64 %rd31, %rd30, %rd10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd40, [%rd17+48], %rd15, %rd31;
-; SM60-NEXT: setp.ne.b64 %p7, %rd40, %rd15;
+; SM60-NEXT: max.u64 %rd30, %rd40, %rd11;
+; SM60-NEXT: sub.s64 %rd31, %rd30, %rd11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
+; SM60-NEXT: mov.b64 %rd40, %rd15;
; SM60-NEXT: @%p7 bra $L__BB239_13;
-; SM60-NEXT: // %bb.14: // %atomicrmw.end67
+; SM60-NEXT: // %bb.14: // %atomicrmw.end55
; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
-; SM60-NEXT: $L__BB239_15: // %atomicrmw.start76
+; SM60-NEXT: $L__BB239_15: // %atomicrmw.start62
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: max.u64 %rd32, %rd41, %rd14;
-; SM60-NEXT: sub.s64 %rd33, %rd32, %rd14;
+; SM60-NEXT: max.u64 %rd32, %rd41, %rd12;
+; SM60-NEXT: sub.s64 %rd33, %rd32, %rd12;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
; SM60-NEXT: mov.b64 %rd41, %rd16;
; SM60-NEXT: @%p8 bra $L__BB239_15;
-; SM60-NEXT: // %bb.16: // %atomicrmw.end75
+; SM60-NEXT: // %bb.16: // %atomicrmw.end61
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd40, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
@@ -9913,20 +9931,20 @@ define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB240_1: // %atomicrmw.start
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB240_1;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
; SM60-NOFTZ-ALLOW-LABEL: fadd_acq_rel_v1f32_global_cta(
@@ -10350,20 +10368,20 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB244_1: // %atomicrmw.start
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r3, %r4, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB244_1;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
; SM60-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v1f32_global_cta(
@@ -10373,9 +10391,9 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
-; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r2, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r3, [%rd1], %r2;
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r3;
@@ -10388,9 +10406,9 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
-; SM60-FTZ-DISALLOW-NEXT: membar.cta;
; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r2, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r3, [%rd1], %r2;
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r3;
@@ -10403,9 +10421,9 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
-; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r2, %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r3, [%rd1], %r2;
; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r3;
@@ -10457,14 +10475,14 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
-; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
-; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r3, %r1;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r4, [%rd1], %r3;
-; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r5, %r2;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r6, [%rd1+4], %r5;
+; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r3, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r4, %r1;
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r5, [%rd1], %r4;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r6, [%rd1+4], %r3;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
; SM60-FTZ-DISALLOW-LABEL: fsub_acq_rel_v2f32_global_cta(
@@ -10474,14 +10492,14 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
-; SM60-FTZ-DISALLOW-NEXT: membar.cta;
; SM60-FTZ-DISALLOW-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
-; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r3, %r1;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r4, [%rd1], %r3;
-; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r5, %r2;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r6, [%rd1+4], %r5;
+; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r3, %r2;
+; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r4, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r5, [%rd1], %r4;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r6, [%rd1+4], %r3;
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM60-FTZ-DISALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM60-FTZ-DISALLOW-NEXT: ret;
;
; SM60-FTZ-ALLOW-LABEL: fsub_acq_rel_v2f32_global_cta(
@@ -10491,14 +10509,14 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
-; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
-; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r3, %r1;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r4, [%rd1], %r3;
-; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r5, %r2;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r6, [%rd1+4], %r5;
+; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r3, %r2;
+; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r4, %r1;
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r5, [%rd1], %r4;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r6, [%rd1+4], %r3;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
@@ -10508,57 +10526,61 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM60-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
; SM60-NOFTZ-DISALLOW: {
; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
-; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<17>;
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<22>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v4f32_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: neg.f32 %r2, %r6;
+; SM60-NOFTZ-DISALLOW-NEXT: neg.f32 %r1, %r5;
+; SM60-NOFTZ-DISALLOW-NEXT: neg.f32 %r4, %r8;
+; SM60-NOFTZ-DISALLOW-NEXT: neg.f32 %r3, %r7;
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r13, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB246_1: // %atomicrmw.start
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r5, %r9, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r6, %r10, %r2;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r9, %r13, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r10, %r14, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r10;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r9;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r14;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r13, %rd11;
; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB246_1;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r15, %rd12;
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB246_3: // %atomicrmw.start2
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r7, %r11, %r3;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r7;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r8, %r12, %r4;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r11, %r15, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r12, %r16, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r12;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r16;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r15, %rd21;
; SM60-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB246_3;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
; SM60-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
@@ -10568,18 +10590,18 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
-; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
-; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r5, %r1;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r6, [%rd1], %r5;
+; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r5, %r4;
+; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r6, %r3;
; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r7, %r2;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r8, [%rd1+4], %r7;
-; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r9, %r3;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r10, [%rd1+8], %r9;
-; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r11, %r4;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r12, [%rd1+12], %r11;
+; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r8, %r1;
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r9, [%rd1], %r8;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r10, [%rd1+4], %r7;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r11, [%rd1+8], %r6;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r12, [%rd1+12], %r5;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
; SM60-FTZ-DISALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
@@ -10589,18 +10611,18 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
-; SM60-FTZ-DISALLOW-NEXT: membar.cta;
; SM60-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
-; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r5, %r1;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r6, [%rd1], %r5;
+; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r5, %r4;
+; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r6, %r3;
; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r7, %r2;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r8, [%rd1+4], %r7;
-; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r9, %r3;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r10, [%rd1+8], %r9;
-; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r11, %r4;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r12, [%rd1+12], %r11;
+; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r8, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r9, [%rd1], %r8;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r10, [%rd1+4], %r7;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r11, [%rd1+8], %r6;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r12, [%rd1+12], %r5;
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM60-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM60-FTZ-DISALLOW-NEXT: ret;
;
; SM60-FTZ-ALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
@@ -10610,18 +10632,18 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
-; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
-; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r5, %r1;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r6, [%rd1], %r5;
+; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r5, %r4;
+; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r6, %r3;
; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r7, %r2;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r8, [%rd1+4], %r7;
-; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r9, %r3;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r10, [%rd1+8], %r9;
-; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r11, %r4;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r12, [%rd1+12], %r11;
+; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r8, %r1;
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r9, [%rd1], %r8;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r10, [%rd1+4], %r7;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r11, [%rd1+8], %r6;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r12, [%rd1+12], %r5;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
@@ -10631,101 +10653,109 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
; SM60-NOFTZ-DISALLOW: {
; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<5>;
-; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<25>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<33>;
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<42>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r9, %r10, %r11, %r12}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
+; SM60-NOFTZ-DISALLOW-NEXT: neg.f32 %r4, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: neg.f32 %r3, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: neg.f32 %r2, %r10;
+; SM60-NOFTZ-DISALLOW-NEXT: neg.f32 %r1, %r9;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r13, %r14, %r15, %r16}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: neg.f32 %r6, %r14;
+; SM60-NOFTZ-DISALLOW-NEXT: neg.f32 %r5, %r13;
+; SM60-NOFTZ-DISALLOW-NEXT: neg.f32 %r8, %r16;
+; SM60-NOFTZ-DISALLOW-NEXT: neg.f32 %r7, %r15;
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r25, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB247_1: // %atomicrmw.start
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r9, %r17, %r5;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r10, %r18, %r6;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r17, %r25, %r5;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r18, %r26, %r6;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r18;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r17;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r26;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r25, %rd11;
; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB247_1;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r27, %rd12;
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB247_3: // %atomicrmw.start6
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r11, %r19, %r7;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r11;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r12, %r20, %r8;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r19, %r27, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r19;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r20, %r28, %r8;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r20;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r19;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r28;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r27, %rd21;
; SM60-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB247_3;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r29, %rd22;
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB247_5: // %atomicrmw.start16
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r13, %r21, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd23, %r13;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r14, %r22, %r2;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r21, %r29, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd23, %r21;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r22, %r30, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd24, %r22;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd25, %rd24, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd26, %rd23, %rd25;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd27, %r21;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd27, %r29;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd28, %r30;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd29, %rd28, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd30, %rd27, %rd29;
; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r29, %rd31;
; SM60-NOFTZ-DISALLOW-NEXT: @%p3 bra $L__BB247_5;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r31, %rd32;
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB247_7: // %atomicrmw.start22
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r15, %r23, %r3;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd33, %r15;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r16, %r24, %r4;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r23, %r31, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd33, %r23;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r24, %r32, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd34, %r24;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd35, %rd34, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd36, %rd33, %rd35;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd37, %r23;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd37, %r31;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd38, %r32;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd39, %rd38, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd40, %rd37, %rd39;
; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r31, %rd41;
; SM60-NOFTZ-DISALLOW-NEXT: @%p4 bra $L__BB247_7;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM60-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
; SM60-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
@@ -10735,28 +10765,28 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
-; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
-; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r9, %r5;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r10, [%rd1], %r9;
-; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r11, %r6;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r12, [%rd1+4], %r11;
-; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r13, %r7;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r14, [%rd1+8], %r13;
-; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r15, %r8;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r16, [%rd1+12], %r15;
-; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r17, %r1;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r18, [%rd1+16], %r17;
-; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r19, %r2;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r20, [%rd1+20], %r19;
-; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r21, %r3;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r22, [%rd1+24], %r21;
-; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r23, %r4;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r24, [%rd1+28], %r23;
+; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r5, %r4;
+; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r6, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r7, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r8, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r9, %r10, %r11, %r12}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r13, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r14, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r15, %r10;
+; SM60-NOFTZ-ALLOW-NEXT: neg.f32 %r16, %r9;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r17, [%rd1], %r16;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r18, [%rd1+4], %r15;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r19, [%rd1+8], %r14;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r20, [%rd1+12], %r13;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r21, [%rd1+16], %r8;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r22, [%rd1+20], %r7;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r23, [%rd1+24], %r6;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r24, [%rd1+28], %r5;
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
-; SM60-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM60-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
; SM60-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
@@ -10766,28 +10796,28 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
-; SM60-FTZ-DISALLOW-NEXT: membar.cta;
; SM60-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
-; SM60-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
-; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r9, %r5;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r10, [%rd1], %r9;
-; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r11, %r6;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r12, [%rd1+4], %r11;
-; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r13, %r7;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r14, [%rd1+8], %r13;
-; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r15, %r8;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r16, [%rd1+12], %r15;
-; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r17, %r1;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r18, [%rd1+16], %r17;
-; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r19, %r2;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r20, [%rd1+20], %r19;
-; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r21, %r3;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r22, [%rd1+24], %r21;
-; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r23, %r4;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r24, [%rd1+28], %r23;
+; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r5, %r4;
+; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r6, %r3;
+; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r7, %r2;
+; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r8, %r1;
+; SM60-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r9, %r10, %r11, %r12}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r13, %r12;
+; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r14, %r11;
+; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r15, %r10;
+; SM60-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r16, %r9;
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
-; SM60-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r17, [%rd1], %r16;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r18, [%rd1+4], %r15;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r19, [%rd1+8], %r14;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r20, [%rd1+12], %r13;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r21, [%rd1+16], %r8;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r22, [%rd1+20], %r7;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r23, [%rd1+24], %r6;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r24, [%rd1+28], %r5;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM60-FTZ-DISALLOW-NEXT: ret;
;
; SM60-FTZ-ALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
@@ -10797,28 +10827,28 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
-; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
-; SM60-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
-; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r9, %r5;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r10, [%rd1], %r9;
-; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r11, %r6;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r12, [%rd1+4], %r11;
-; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r13, %r7;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r14, [%rd1+8], %r13;
-; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r15, %r8;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r16, [%rd1+12], %r15;
-; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r17, %r1;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r18, [%rd1+16], %r17;
-; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r19, %r2;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r20, [%rd1+20], %r19;
-; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r21, %r3;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r22, [%rd1+24], %r21;
-; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r23, %r4;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r24, [%rd1+28], %r23;
+; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r5, %r4;
+; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r6, %r3;
+; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r7, %r2;
+; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r8, %r1;
+; SM60-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r9, %r10, %r11, %r12}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r13, %r12;
+; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r14, %r11;
+; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r15, %r10;
+; SM60-FTZ-ALLOW-NEXT: neg.ftz.f32 %r16, %r9;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r17, [%rd1], %r16;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r18, [%rd1+4], %r15;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r19, [%rd1+8], %r14;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r20, [%rd1+12], %r13;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r21, [%rd1+16], %r8;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r22, [%rd1+20], %r7;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r23, [%rd1+24], %r6;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r24, [%rd1+28], %r5;
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
-; SM60-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM60-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
@@ -10832,20 +10862,20 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB248_1: // %atomicrmw.start
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: min.f32 %r3, %r4, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: min.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB248_1;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
; SM60-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v1f32_global_cta(
@@ -10855,20 +10885,20 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NOFTZ-ALLOW-NEXT: $L__BB248_1: // %atomicrmw.start
; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-ALLOW-NEXT: min.f32 %r3, %r4, %r1;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: min.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM60-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB248_1;
; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
; SM60-FTZ-DISALLOW-LABEL: fmin_acq_rel_v1f32_global_cta(
@@ -10878,20 +10908,20 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-FTZ-DISALLOW-NEXT: $L__BB248_1: // %atomicrmw.start
; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-DISALLOW-NEXT: min.ftz.f32 %r3, %r4, %r1;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM60-FTZ-DISALLOW-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM60-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB248_1;
; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-FTZ-DISALLOW-NEXT: ret;
;
; SM60-FTZ-ALLOW-LABEL: fmin_acq_rel_v1f32_global_cta(
@@ -10901,20 +10931,20 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-FTZ-ALLOW-NEXT: $L__BB248_1: // %atomicrmw.start
; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-ALLOW-NEXT: min.ftz.f32 %r3, %r4, %r1;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM60-FTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM60-FTZ-ALLOW-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM60-FTZ-ALLOW-NEXT: @%p1 bra $L__BB248_1;
; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -11704,20 +11734,20 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB252_1: // %atomicrmw.start
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: max.f32 %r3, %r4, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: max.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB252_1;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
; SM60-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v1f32_global_cta(
@@ -11727,20 +11757,20 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NOFTZ-ALLOW-NEXT: $L__BB252_1: // %atomicrmw.start
; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-ALLOW-NEXT: max.f32 %r3, %r4, %r1;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: max.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM60-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB252_1;
; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
; SM60-FTZ-DISALLOW-LABEL: fmax_acq_rel_v1f32_global_cta(
@@ -11750,20 +11780,20 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-FTZ-DISALLOW-NEXT: $L__BB252_1: // %atomicrmw.start
; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-DISALLOW-NEXT: max.ftz.f32 %r3, %r4, %r1;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM60-FTZ-DISALLOW-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM60-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB252_1;
; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-FTZ-DISALLOW-NEXT: ret;
;
; SM60-FTZ-ALLOW-LABEL: fmax_acq_rel_v1f32_global_cta(
@@ -11773,20 +11803,20 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-FTZ-ALLOW-NEXT: $L__BB252_1: // %atomicrmw.start
; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-ALLOW-NEXT: max.ftz.f32 %r3, %r4, %r1;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM60-FTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM60-FTZ-ALLOW-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM60-FTZ-ALLOW-NEXT: @%p1 bra $L__BB252_1;
; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -12576,26 +12606,26 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB256_1: // %atomicrmw.start
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
-; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r7, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT: min.f32 %r4, %r7, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: min.f32 %r4, %r7, %r2;
; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p3, %r7, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
; SM60-NOFTZ-DISALLOW-NEXT: @%p4 bra $L__BB256_1;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
; SM60-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v1f32_global_cta(
@@ -12605,26 +12635,26 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
; SM60-NOFTZ-ALLOW-NEXT: $L__BB256_1: // %atomicrmw.start
; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NOFTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
-; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r7, %r1;
-; SM60-NOFTZ-ALLOW-NEXT: min.f32 %r4, %r7, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: min.f32 %r4, %r7, %r2;
; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM60-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p3, %r7, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r7, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
; SM60-NOFTZ-ALLOW-NEXT: @%p4 bra $L__BB256_1;
; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
; SM60-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v1f32_global_cta(
@@ -12634,26 +12664,26 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
; SM60-FTZ-DISALLOW-NEXT: $L__BB256_1: // %atomicrmw.start
; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-FTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
-; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r1;
-; SM60-FTZ-DISALLOW-NEXT: min.ftz.f32 %r4, %r7, %r1;
+; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM60-FTZ-DISALLOW-NEXT: min.ftz.f32 %r4, %r7, %r2;
; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM60-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r1;
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r7, %r2;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
; SM60-FTZ-DISALLOW-NEXT: @%p4 bra $L__BB256_1;
; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-FTZ-DISALLOW-NEXT: ret;
;
; SM60-FTZ-ALLOW-LABEL: fminimum_acq_rel_v1f32_global_cta(
@@ -12663,26 +12693,26 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
; SM60-FTZ-ALLOW-NEXT: $L__BB256_1: // %atomicrmw.start
; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-FTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
-; SM60-FTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r1;
-; SM60-FTZ-ALLOW-NEXT: min.ftz.f32 %r4, %r7, %r1;
+; SM60-FTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM60-FTZ-ALLOW-NEXT: min.ftz.f32 %r4, %r7, %r2;
; SM60-FTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM60-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r1;
+; SM60-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
; SM60-FTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM60-FTZ-ALLOW-NEXT: mov.b32 %r7, %r2;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
; SM60-FTZ-ALLOW-NEXT: @%p4 bra $L__BB256_1;
; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -13808,26 +13838,26 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB260_1: // %atomicrmw.start
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
-; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r7, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT: max.f32 %r4, %r7, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: max.f32 %r4, %r7, %r2;
; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p3, %r7, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
; SM60-NOFTZ-DISALLOW-NEXT: @%p4 bra $L__BB260_1;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
; SM60-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v1f32_global_cta(
@@ -13837,26 +13867,26 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
; SM60-NOFTZ-ALLOW-NEXT: $L__BB260_1: // %atomicrmw.start
; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NOFTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
-; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r7, %r1;
-; SM60-NOFTZ-ALLOW-NEXT: max.f32 %r4, %r7, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: max.f32 %r4, %r7, %r2;
; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM60-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p3, %r7, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r7, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
; SM60-NOFTZ-ALLOW-NEXT: @%p4 bra $L__BB260_1;
; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
; SM60-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v1f32_global_cta(
@@ -13866,26 +13896,26 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
; SM60-FTZ-DISALLOW-NEXT: $L__BB260_1: // %atomicrmw.start
; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-FTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
-; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r1;
-; SM60-FTZ-DISALLOW-NEXT: max.ftz.f32 %r4, %r7, %r1;
+; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM60-FTZ-DISALLOW-NEXT: max.ftz.f32 %r4, %r7, %r2;
; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM60-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r1;
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r7, %r2;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
; SM60-FTZ-DISALLOW-NEXT: @%p4 bra $L__BB260_1;
; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-FTZ-DISALLOW-NEXT: ret;
;
; SM60-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v1f32_global_cta(
@@ -13895,26 +13925,26 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
; SM60-FTZ-ALLOW-NEXT: $L__BB260_1: // %atomicrmw.start
; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-FTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
-; SM60-FTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r1;
-; SM60-FTZ-ALLOW-NEXT: max.ftz.f32 %r4, %r7, %r1;
+; SM60-FTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM60-FTZ-ALLOW-NEXT: max.ftz.f32 %r4, %r7, %r2;
; SM60-FTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM60-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r1;
+; SM60-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
; SM60-FTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM60-FTZ-ALLOW-NEXT: mov.b32 %r7, %r2;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
; SM60-FTZ-ALLOW-NEXT: @%p4 bra $L__BB260_1;
; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -15126,9 +15156,9 @@ define <1 x double> @fsub_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f64_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1f64_global_cta_param_1];
; SM60-NEXT: neg.f64 %rd3, %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: atom.cta.global.add.f64 %rd4, [%rd1], %rd3;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd4;
@@ -15144,14 +15174,14 @@ define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f64_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fsub_acq_rel_v2f64_global_cta_param_1];
-; SM60-NEXT: neg.f64 %rd4, %rd2;
-; SM60-NEXT: atom.cta.global.add.f64 %rd5, [%rd1], %rd4;
-; SM60-NEXT: neg.f64 %rd6, %rd3;
-; SM60-NEXT: atom.cta.global.add.f64 %rd7, [%rd1+8], %rd6;
+; SM60-NEXT: neg.f64 %rd4, %rd3;
+; SM60-NEXT: neg.f64 %rd5, %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: atom.cta.global.add.f64 %rd6, [%rd1], %rd5;
+; SM60-NEXT: atom.cta.global.add.f64 %rd7, [%rd1+8], %rd4;
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd7};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -15164,20 +15194,20 @@ define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f64_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fsub_acq_rel_v4f64_global_cta_param_1];
-; SM60-NEXT: neg.f64 %rd6, %rd4;
-; SM60-NEXT: atom.cta.global.add.f64 %rd7, [%rd1], %rd6;
-; SM60-NEXT: neg.f64 %rd8, %rd5;
-; SM60-NEXT: atom.cta.global.add.f64 %rd9, [%rd1+8], %rd8;
-; SM60-NEXT: neg.f64 %rd10, %rd2;
-; SM60-NEXT: atom.cta.global.add.f64 %rd11, [%rd1+16], %rd10;
-; SM60-NEXT: neg.f64 %rd12, %rd3;
-; SM60-NEXT: atom.cta.global.add.f64 %rd13, [%rd1+24], %rd12;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd11, %rd13};
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd7, %rd9};
+; SM60-NEXT: neg.f64 %rd4, %rd3;
+; SM60-NEXT: neg.f64 %rd5, %rd2;
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [fsub_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: neg.f64 %rd8, %rd7;
+; SM60-NEXT: neg.f64 %rd9, %rd6;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: atom.cta.global.add.f64 %rd10, [%rd1], %rd9;
+; SM60-NEXT: atom.cta.global.add.f64 %rd11, [%rd1+8], %rd8;
+; SM60-NEXT: atom.cta.global.add.f64 %rd12, [%rd1+16], %rd5;
+; SM60-NEXT: atom.cta.global.add.f64 %rd13, [%rd1+24], %rd4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
ret <4 x double> %retval
@@ -15190,32 +15220,32 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f64_global_cta_param_0];
-; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fsub_acq_rel_v8f64_global_cta_param_1+48];
-; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
-; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [fsub_acq_rel_v8f64_global_cta_param_1];
-; SM60-NEXT: neg.f64 %rd10, %rd8;
-; SM60-NEXT: atom.cta.global.add.f64 %rd11, [%rd1], %rd10;
-; SM60-NEXT: neg.f64 %rd12, %rd9;
-; SM60-NEXT: atom.cta.global.add.f64 %rd13, [%rd1+8], %rd12;
-; SM60-NEXT: neg.f64 %rd14, %rd6;
-; SM60-NEXT: atom.cta.global.add.f64 %rd15, [%rd1+16], %rd14;
-; SM60-NEXT: neg.f64 %rd16, %rd7;
-; SM60-NEXT: atom.cta.global.add.f64 %rd17, [%rd1+24], %rd16;
-; SM60-NEXT: neg.f64 %rd18, %rd4;
-; SM60-NEXT: atom.cta.global.add.f64 %rd19, [%rd1+32], %rd18;
-; SM60-NEXT: neg.f64 %rd20, %rd5;
-; SM60-NEXT: atom.cta.global.add.f64 %rd21, [%rd1+40], %rd20;
-; SM60-NEXT: neg.f64 %rd22, %rd2;
-; SM60-NEXT: atom.cta.global.add.f64 %rd23, [%rd1+48], %rd22;
-; SM60-NEXT: neg.f64 %rd24, %rd3;
-; SM60-NEXT: atom.cta.global.add.f64 %rd25, [%rd1+56], %rd24;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd23, %rd25};
-; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd19, %rd21};
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd15, %rd17};
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd11, %rd13};
+; SM60-NEXT: neg.f64 %rd4, %rd3;
+; SM60-NEXT: neg.f64 %rd5, %rd2;
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: neg.f64 %rd8, %rd7;
+; SM60-NEXT: neg.f64 %rd9, %rd6;
+; SM60-NEXT: ld.param.v2.b64 {%rd10, %rd11}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: neg.f64 %rd12, %rd11;
+; SM60-NEXT: neg.f64 %rd13, %rd10;
+; SM60-NEXT: ld.param.v2.b64 {%rd14, %rd15}, [fsub_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: neg.f64 %rd16, %rd15;
+; SM60-NEXT: neg.f64 %rd17, %rd14;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: atom.cta.global.add.f64 %rd18, [%rd1], %rd17;
+; SM60-NEXT: atom.cta.global.add.f64 %rd19, [%rd1+8], %rd16;
+; SM60-NEXT: atom.cta.global.add.f64 %rd20, [%rd1+16], %rd13;
+; SM60-NEXT: atom.cta.global.add.f64 %rd21, [%rd1+24], %rd12;
+; SM60-NEXT: atom.cta.global.add.f64 %rd22, [%rd1+32], %rd9;
+; SM60-NEXT: atom.cta.global.add.f64 %rd23, [%rd1+40], %rd8;
+; SM60-NEXT: atom.cta.global.add.f64 %rd24, [%rd1+48], %rd5;
+; SM60-NEXT: atom.cta.global.add.f64 %rd25, [%rd1+56], %rd4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd24, %rd25};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd22, %rd23};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd20, %rd21};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -15228,20 +15258,20 @@ define <1 x double> @fmin_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-NEXT: .reg .b64 %rd<6>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd5, [%rd3];
+; SM60-NEXT: ld.volatile.global.b64 %rd5, [%rd2];
; SM60-NEXT: $L__BB272_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: min.f64 %rd4, %rd5, %rd1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
-; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
-; SM60-NEXT: mov.b64 %rd5, %rd2;
+; SM60-NEXT: min.f64 %rd4, %rd5, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd1, [%rd2], %rd5, %rd4;
+; SM60-NEXT: setp.ne.b64 %p1, %rd1, %rd5;
+; SM60-NEXT: mov.b64 %rd5, %rd1;
; SM60-NEXT: @%p1 bra $L__BB272_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -15256,27 +15286,27 @@ define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd5, [fmin_acq_rel_v2f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [fmin_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v2f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd5];
; SM60-NEXT: $L__BB273_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd3, %rd8;
-; SM60-NEXT: min.f64 %rd6, %rd3, %rd2;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd5], %rd3, %rd6;
-; SM60-NEXT: setp.ne.b64 %p1, %rd8, %rd3;
+; SM60-NEXT: min.f64 %rd6, %rd8, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM60-NEXT: mov.b64 %rd8, %rd3;
; SM60-NEXT: @%p1 bra $L__BB273_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd5+8];
-; SM60-NEXT: $L__BB273_3: // %atomicrmw.start4
+; SM60-NEXT: $L__BB273_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: min.f64 %rd7, %rd9, %rd1;
+; SM60-NEXT: min.f64 %rd7, %rd9, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
; SM60-NEXT: mov.b64 %rd9, %rd4;
; SM60-NEXT: @%p2 bra $L__BB273_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end3
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd8, %rd4};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -15291,46 +15321,46 @@ define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd9, [fmin_acq_rel_v4f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v4f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd9];
; SM60-NEXT: $L__BB274_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd4, %rd14;
-; SM60-NEXT: min.f64 %rd10, %rd4, %rd3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd9], %rd4, %rd10;
-; SM60-NEXT: setp.ne.b64 %p1, %rd14, %rd4;
+; SM60-NEXT: mov.b64 %rd5, %rd14;
+; SM60-NEXT: min.f64 %rd10, %rd5, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
; SM60-NEXT: @%p1 bra $L__BB274_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd9+8];
-; SM60-NEXT: $L__BB274_3: // %atomicrmw.start8
+; SM60-NEXT: $L__BB274_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd15;
-; SM60-NEXT: min.f64 %rd11, %rd5, %rd2;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd9+8], %rd5, %rd11;
-; SM60-NEXT: setp.ne.b64 %p2, %rd15, %rd5;
+; SM60-NEXT: mov.b64 %rd6, %rd15;
+; SM60-NEXT: min.f64 %rd11, %rd6, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
; SM60-NEXT: @%p2 bra $L__BB274_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end7
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd9+16];
-; SM60-NEXT: $L__BB274_5: // %atomicrmw.start19
+; SM60-NEXT: $L__BB274_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd7, %rd16;
-; SM60-NEXT: min.f64 %rd12, %rd7, %rd1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd9+16], %rd7, %rd12;
-; SM60-NEXT: setp.ne.b64 %p3, %rd16, %rd7;
+; SM60-NEXT: min.f64 %rd12, %rd16, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM60-NEXT: mov.b64 %rd16, %rd7;
; SM60-NEXT: @%p3 bra $L__BB274_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end18
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd9+24];
-; SM60-NEXT: $L__BB274_7: // %atomicrmw.start26
+; SM60-NEXT: $L__BB274_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: min.f64 %rd13, %rd17, %rd6;
+; SM60-NEXT: min.f64 %rd13, %rd17, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
; SM60-NEXT: mov.b64 %rd17, %rd8;
; SM60-NEXT: @%p4 bra $L__BB274_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd16, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
@@ -15346,84 +15376,84 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd17, [fmin_acq_rel_v8f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [fmin_acq_rel_v8f64_global_cta_param_1+48];
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [fmin_acq_rel_v8f64_global_cta_param_1+32];
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [fmin_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmin_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmin_acq_rel_v8f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd17];
; SM60-NEXT: $L__BB275_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd26;
-; SM60-NEXT: min.f64 %rd18, %rd5, %rd4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd17], %rd5, %rd18;
-; SM60-NEXT: setp.ne.b64 %p1, %rd26, %rd5;
+; SM60-NEXT: mov.b64 %rd7, %rd26;
+; SM60-NEXT: min.f64 %rd18, %rd7, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM60-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
; SM60-NEXT: @%p1 bra $L__BB275_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd17+8];
-; SM60-NEXT: $L__BB275_3: // %atomicrmw.start12
+; SM60-NEXT: $L__BB275_3: // %atomicrmw.start10
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd6, %rd27;
-; SM60-NEXT: min.f64 %rd19, %rd6, %rd3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd17+8], %rd6, %rd19;
-; SM60-NEXT: setp.ne.b64 %p2, %rd27, %rd6;
+; SM60-NEXT: mov.b64 %rd8, %rd27;
+; SM60-NEXT: min.f64 %rd19, %rd8, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
; SM60-NEXT: @%p2 bra $L__BB275_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end11
+; SM60-NEXT: // %bb.4: // %atomicrmw.end9
; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd17+16];
-; SM60-NEXT: $L__BB275_5: // %atomicrmw.start23
+; SM60-NEXT: $L__BB275_5: // %atomicrmw.start19
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd8, %rd28;
-; SM60-NEXT: min.f64 %rd20, %rd8, %rd2;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd17+16], %rd8, %rd20;
-; SM60-NEXT: setp.ne.b64 %p3, %rd28, %rd8;
+; SM60-NEXT: mov.b64 %rd9, %rd28;
+; SM60-NEXT: min.f64 %rd20, %rd9, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM60-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
; SM60-NEXT: @%p3 bra $L__BB275_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end22
+; SM60-NEXT: // %bb.6: // %atomicrmw.end18
; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd17+24];
-; SM60-NEXT: $L__BB275_7: // %atomicrmw.start30
+; SM60-NEXT: $L__BB275_7: // %atomicrmw.start24
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd9, %rd29;
-; SM60-NEXT: min.f64 %rd21, %rd9, %rd7;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd29, [%rd17+24], %rd9, %rd21;
-; SM60-NEXT: setp.ne.b64 %p4, %rd29, %rd9;
+; SM60-NEXT: mov.b64 %rd10, %rd29;
+; SM60-NEXT: min.f64 %rd21, %rd10, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
; SM60-NEXT: @%p4 bra $L__BB275_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end29
+; SM60-NEXT: // %bb.8: // %atomicrmw.end23
; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd17+32];
-; SM60-NEXT: $L__BB275_9: // %atomicrmw.start45
+; SM60-NEXT: $L__BB275_9: // %atomicrmw.start37
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd12, %rd30;
-; SM60-NEXT: min.f64 %rd22, %rd12, %rd1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd17+32], %rd12, %rd22;
-; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd12;
+; SM60-NEXT: mov.b64 %rd13, %rd30;
+; SM60-NEXT: min.f64 %rd22, %rd13, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
; SM60-NEXT: @%p5 bra $L__BB275_9;
-; SM60-NEXT: // %bb.10: // %atomicrmw.end44
+; SM60-NEXT: // %bb.10: // %atomicrmw.end36
; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd17+40];
-; SM60-NEXT: $L__BB275_11: // %atomicrmw.start52
+; SM60-NEXT: $L__BB275_11: // %atomicrmw.start42
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd13, %rd31;
-; SM60-NEXT: min.f64 %rd23, %rd13, %rd11;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd17+40], %rd13, %rd23;
-; SM60-NEXT: setp.ne.b64 %p6, %rd31, %rd13;
+; SM60-NEXT: mov.b64 %rd14, %rd31;
+; SM60-NEXT: min.f64 %rd23, %rd14, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
; SM60-NEXT: @%p6 bra $L__BB275_11;
-; SM60-NEXT: // %bb.12: // %atomicrmw.end51
+; SM60-NEXT: // %bb.12: // %atomicrmw.end41
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd17+48];
-; SM60-NEXT: $L__BB275_13: // %atomicrmw.start63
+; SM60-NEXT: $L__BB275_13: // %atomicrmw.start51
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd15, %rd32;
-; SM60-NEXT: min.f64 %rd24, %rd15, %rd10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd32, [%rd17+48], %rd15, %rd24;
-; SM60-NEXT: setp.ne.b64 %p7, %rd32, %rd15;
+; SM60-NEXT: min.f64 %rd24, %rd32, %rd11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM60-NEXT: mov.b64 %rd32, %rd15;
; SM60-NEXT: @%p7 bra $L__BB275_13;
-; SM60-NEXT: // %bb.14: // %atomicrmw.end62
+; SM60-NEXT: // %bb.14: // %atomicrmw.end50
; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd17+56];
-; SM60-NEXT: $L__BB275_15: // %atomicrmw.start70
+; SM60-NEXT: $L__BB275_15: // %atomicrmw.start56
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: min.f64 %rd25, %rd33, %rd14;
+; SM60-NEXT: min.f64 %rd25, %rd33, %rd12;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
; SM60-NEXT: mov.b64 %rd33, %rd16;
; SM60-NEXT: @%p8 bra $L__BB275_15;
-; SM60-NEXT: // %bb.16: // %atomicrmw.end69
+; SM60-NEXT: // %bb.16: // %atomicrmw.end55
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd32, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
@@ -15439,20 +15469,20 @@ define <1 x double> @fmax_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-NEXT: .reg .b64 %rd<6>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd5, [%rd3];
+; SM60-NEXT: ld.volatile.global.b64 %rd5, [%rd2];
; SM60-NEXT: $L__BB276_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: max.f64 %rd4, %rd5, %rd1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
-; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
-; SM60-NEXT: mov.b64 %rd5, %rd2;
+; SM60-NEXT: max.f64 %rd4, %rd5, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd1, [%rd2], %rd5, %rd4;
+; SM60-NEXT: setp.ne.b64 %p1, %rd1, %rd5;
+; SM60-NEXT: mov.b64 %rd5, %rd1;
; SM60-NEXT: @%p1 bra $L__BB276_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -15467,27 +15497,27 @@ define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd5, [fmax_acq_rel_v2f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [fmax_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v2f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd5];
; SM60-NEXT: $L__BB277_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd3, %rd8;
-; SM60-NEXT: max.f64 %rd6, %rd3, %rd2;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd5], %rd3, %rd6;
-; SM60-NEXT: setp.ne.b64 %p1, %rd8, %rd3;
+; SM60-NEXT: max.f64 %rd6, %rd8, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM60-NEXT: mov.b64 %rd8, %rd3;
; SM60-NEXT: @%p1 bra $L__BB277_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd5+8];
-; SM60-NEXT: $L__BB277_3: // %atomicrmw.start4
+; SM60-NEXT: $L__BB277_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: max.f64 %rd7, %rd9, %rd1;
+; SM60-NEXT: max.f64 %rd7, %rd9, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
; SM60-NEXT: mov.b64 %rd9, %rd4;
; SM60-NEXT: @%p2 bra $L__BB277_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end3
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd8, %rd4};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -15502,46 +15532,46 @@ define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd9, [fmax_acq_rel_v4f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v4f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd9];
; SM60-NEXT: $L__BB278_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd4, %rd14;
-; SM60-NEXT: max.f64 %rd10, %rd4, %rd3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd9], %rd4, %rd10;
-; SM60-NEXT: setp.ne.b64 %p1, %rd14, %rd4;
+; SM60-NEXT: mov.b64 %rd5, %rd14;
+; SM60-NEXT: max.f64 %rd10, %rd5, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
; SM60-NEXT: @%p1 bra $L__BB278_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd9+8];
-; SM60-NEXT: $L__BB278_3: // %atomicrmw.start8
+; SM60-NEXT: $L__BB278_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd15;
-; SM60-NEXT: max.f64 %rd11, %rd5, %rd2;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd9+8], %rd5, %rd11;
-; SM60-NEXT: setp.ne.b64 %p2, %rd15, %rd5;
+; SM60-NEXT: mov.b64 %rd6, %rd15;
+; SM60-NEXT: max.f64 %rd11, %rd6, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
; SM60-NEXT: @%p2 bra $L__BB278_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end7
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd9+16];
-; SM60-NEXT: $L__BB278_5: // %atomicrmw.start19
+; SM60-NEXT: $L__BB278_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd7, %rd16;
-; SM60-NEXT: max.f64 %rd12, %rd7, %rd1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd9+16], %rd7, %rd12;
-; SM60-NEXT: setp.ne.b64 %p3, %rd16, %rd7;
+; SM60-NEXT: max.f64 %rd12, %rd16, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM60-NEXT: mov.b64 %rd16, %rd7;
; SM60-NEXT: @%p3 bra $L__BB278_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end18
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd9+24];
-; SM60-NEXT: $L__BB278_7: // %atomicrmw.start26
+; SM60-NEXT: $L__BB278_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: max.f64 %rd13, %rd17, %rd6;
+; SM60-NEXT: max.f64 %rd13, %rd17, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
; SM60-NEXT: mov.b64 %rd17, %rd8;
; SM60-NEXT: @%p4 bra $L__BB278_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd16, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
@@ -15557,84 +15587,84 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd17, [fmax_acq_rel_v8f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [fmax_acq_rel_v8f64_global_cta_param_1+48];
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [fmax_acq_rel_v8f64_global_cta_param_1+32];
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [fmax_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmax_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmax_acq_rel_v8f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd17];
; SM60-NEXT: $L__BB279_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd26;
-; SM60-NEXT: max.f64 %rd18, %rd5, %rd4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd17], %rd5, %rd18;
-; SM60-NEXT: setp.ne.b64 %p1, %rd26, %rd5;
+; SM60-NEXT: mov.b64 %rd7, %rd26;
+; SM60-NEXT: max.f64 %rd18, %rd7, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM60-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
; SM60-NEXT: @%p1 bra $L__BB279_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd17+8];
-; SM60-NEXT: $L__BB279_3: // %atomicrmw.start12
+; SM60-NEXT: $L__BB279_3: // %atomicrmw.start10
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd6, %rd27;
-; SM60-NEXT: max.f64 %rd19, %rd6, %rd3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd17+8], %rd6, %rd19;
-; SM60-NEXT: setp.ne.b64 %p2, %rd27, %rd6;
+; SM60-NEXT: mov.b64 %rd8, %rd27;
+; SM60-NEXT: max.f64 %rd19, %rd8, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
; SM60-NEXT: @%p2 bra $L__BB279_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end11
+; SM60-NEXT: // %bb.4: // %atomicrmw.end9
; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd17+16];
-; SM60-NEXT: $L__BB279_5: // %atomicrmw.start23
+; SM60-NEXT: $L__BB279_5: // %atomicrmw.start19
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd8, %rd28;
-; SM60-NEXT: max.f64 %rd20, %rd8, %rd2;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd17+16], %rd8, %rd20;
-; SM60-NEXT: setp.ne.b64 %p3, %rd28, %rd8;
+; SM60-NEXT: mov.b64 %rd9, %rd28;
+; SM60-NEXT: max.f64 %rd20, %rd9, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM60-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
; SM60-NEXT: @%p3 bra $L__BB279_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end22
+; SM60-NEXT: // %bb.6: // %atomicrmw.end18
; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd17+24];
-; SM60-NEXT: $L__BB279_7: // %atomicrmw.start30
+; SM60-NEXT: $L__BB279_7: // %atomicrmw.start24
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd9, %rd29;
-; SM60-NEXT: max.f64 %rd21, %rd9, %rd7;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd29, [%rd17+24], %rd9, %rd21;
-; SM60-NEXT: setp.ne.b64 %p4, %rd29, %rd9;
+; SM60-NEXT: mov.b64 %rd10, %rd29;
+; SM60-NEXT: max.f64 %rd21, %rd10, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
; SM60-NEXT: @%p4 bra $L__BB279_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end29
+; SM60-NEXT: // %bb.8: // %atomicrmw.end23
; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd17+32];
-; SM60-NEXT: $L__BB279_9: // %atomicrmw.start45
+; SM60-NEXT: $L__BB279_9: // %atomicrmw.start37
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd12, %rd30;
-; SM60-NEXT: max.f64 %rd22, %rd12, %rd1;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd17+32], %rd12, %rd22;
-; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd12;
+; SM60-NEXT: mov.b64 %rd13, %rd30;
+; SM60-NEXT: max.f64 %rd22, %rd13, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
; SM60-NEXT: @%p5 bra $L__BB279_9;
-; SM60-NEXT: // %bb.10: // %atomicrmw.end44
+; SM60-NEXT: // %bb.10: // %atomicrmw.end36
; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd17+40];
-; SM60-NEXT: $L__BB279_11: // %atomicrmw.start52
+; SM60-NEXT: $L__BB279_11: // %atomicrmw.start42
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd13, %rd31;
-; SM60-NEXT: max.f64 %rd23, %rd13, %rd11;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd17+40], %rd13, %rd23;
-; SM60-NEXT: setp.ne.b64 %p6, %rd31, %rd13;
+; SM60-NEXT: mov.b64 %rd14, %rd31;
+; SM60-NEXT: max.f64 %rd23, %rd14, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
; SM60-NEXT: @%p6 bra $L__BB279_11;
-; SM60-NEXT: // %bb.12: // %atomicrmw.end51
+; SM60-NEXT: // %bb.12: // %atomicrmw.end41
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd17+48];
-; SM60-NEXT: $L__BB279_13: // %atomicrmw.start63
+; SM60-NEXT: $L__BB279_13: // %atomicrmw.start51
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd15, %rd32;
-; SM60-NEXT: max.f64 %rd24, %rd15, %rd10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd32, [%rd17+48], %rd15, %rd24;
-; SM60-NEXT: setp.ne.b64 %p7, %rd32, %rd15;
+; SM60-NEXT: max.f64 %rd24, %rd32, %rd11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM60-NEXT: mov.b64 %rd32, %rd15;
; SM60-NEXT: @%p7 bra $L__BB279_13;
-; SM60-NEXT: // %bb.14: // %atomicrmw.end62
+; SM60-NEXT: // %bb.14: // %atomicrmw.end50
; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd17+56];
-; SM60-NEXT: $L__BB279_15: // %atomicrmw.start70
+; SM60-NEXT: $L__BB279_15: // %atomicrmw.start56
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: max.f64 %rd25, %rd33, %rd14;
+; SM60-NEXT: max.f64 %rd25, %rd33, %rd12;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
; SM60-NEXT: mov.b64 %rd33, %rd16;
; SM60-NEXT: @%p8 bra $L__BB279_15;
-; SM60-NEXT: // %bb.16: // %atomicrmw.end69
+; SM60-NEXT: // %bb.16: // %atomicrmw.end55
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd32, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
@@ -15650,26 +15680,26 @@ define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: .reg .b64 %rd<9>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd3];
+; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd2];
; SM60-NEXT: $L__BB280_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.eq.b64 %p1, %rd8, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd4, %rd8, %rd1, %p1;
-; SM60-NEXT: setp.nan.f64 %p2, %rd8, %rd1;
-; SM60-NEXT: min.f64 %rd5, %rd8, %rd1;
+; SM60-NEXT: selp.f64 %rd4, %rd8, %rd3, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd8, %rd3;
+; SM60-NEXT: min.f64 %rd5, %rd8, %rd3;
; SM60-NEXT: selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
-; SM60-NEXT: setp.eq.f64 %p3, %rd8, %rd1;
+; SM60-NEXT: setp.eq.f64 %p3, %rd8, %rd3;
; SM60-NEXT: selp.f64 %rd7, %rd4, %rd6, %p3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
-; SM60-NEXT: setp.ne.b64 %p4, %rd2, %rd8;
-; SM60-NEXT: mov.b64 %rd8, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd1, [%rd2], %rd8, %rd7;
+; SM60-NEXT: setp.ne.b64 %p4, %rd1, %rd8;
+; SM60-NEXT: mov.b64 %rd8, %rd1;
; SM60-NEXT: @%p4 bra $L__BB280_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -15684,39 +15714,39 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd5, [fminimum_acq_rel_v2f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [fminimum_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v2f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd5];
; SM60-NEXT: $L__BB281_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd3, %rd14;
-; SM60-NEXT: setp.eq.b64 %p1, %rd3, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd6, %rd3, %rd2, %p1;
-; SM60-NEXT: setp.nan.f64 %p2, %rd3, %rd2;
-; SM60-NEXT: min.f64 %rd7, %rd3, %rd2;
+; SM60-NEXT: setp.eq.b64 %p1, %rd14, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd6, %rd14, %rd1, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd14, %rd1;
+; SM60-NEXT: min.f64 %rd7, %rd14, %rd1;
; SM60-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p2;
-; SM60-NEXT: setp.eq.f64 %p3, %rd3, %rd2;
+; SM60-NEXT: setp.eq.f64 %p3, %rd14, %rd1;
; SM60-NEXT: selp.f64 %rd9, %rd6, %rd8, %p3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd5], %rd3, %rd9;
-; SM60-NEXT: setp.ne.b64 %p4, %rd14, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd14, %rd9;
+; SM60-NEXT: setp.ne.b64 %p4, %rd3, %rd14;
+; SM60-NEXT: mov.b64 %rd14, %rd3;
; SM60-NEXT: @%p4 bra $L__BB281_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd5+8];
-; SM60-NEXT: $L__BB281_3: // %atomicrmw.start4
+; SM60-NEXT: $L__BB281_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.eq.b64 %p5, %rd15, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd10, %rd15, %rd1, %p5;
-; SM60-NEXT: setp.nan.f64 %p6, %rd15, %rd1;
-; SM60-NEXT: min.f64 %rd11, %rd15, %rd1;
+; SM60-NEXT: selp.f64 %rd10, %rd15, %rd2, %p5;
+; SM60-NEXT: setp.nan.f64 %p6, %rd15, %rd2;
+; SM60-NEXT: min.f64 %rd11, %rd15, %rd2;
; SM60-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
-; SM60-NEXT: setp.eq.f64 %p7, %rd15, %rd1;
+; SM60-NEXT: setp.eq.f64 %p7, %rd15, %rd2;
; SM60-NEXT: selp.f64 %rd13, %rd10, %rd12, %p7;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd15, %rd13;
; SM60-NEXT: setp.ne.b64 %p8, %rd4, %rd15;
; SM60-NEXT: mov.b64 %rd15, %rd4;
; SM60-NEXT: @%p8 bra $L__BB281_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end3
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd4};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -15731,70 +15761,70 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd9];
; SM60-NEXT: $L__BB282_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd4, %rd26;
-; SM60-NEXT: setp.eq.b64 %p1, %rd4, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd10, %rd4, %rd3, %p1;
-; SM60-NEXT: setp.nan.f64 %p2, %rd4, %rd3;
-; SM60-NEXT: min.f64 %rd11, %rd4, %rd3;
+; SM60-NEXT: mov.b64 %rd5, %rd26;
+; SM60-NEXT: setp.eq.b64 %p1, %rd5, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd10, %rd5, %rd3, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd5, %rd3;
+; SM60-NEXT: min.f64 %rd11, %rd5, %rd3;
; SM60-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p2;
-; SM60-NEXT: setp.eq.f64 %p3, %rd4, %rd3;
+; SM60-NEXT: setp.eq.f64 %p3, %rd5, %rd3;
; SM60-NEXT: selp.f64 %rd13, %rd10, %rd12, %p3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd9], %rd4, %rd13;
-; SM60-NEXT: setp.ne.b64 %p4, %rd26, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd9], %rd5, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd26, %rd5;
; SM60-NEXT: @%p4 bra $L__BB282_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd9+8];
-; SM60-NEXT: $L__BB282_3: // %atomicrmw.start8
+; SM60-NEXT: $L__BB282_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd27;
-; SM60-NEXT: setp.eq.b64 %p5, %rd5, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd14, %rd5, %rd2, %p5;
-; SM60-NEXT: setp.nan.f64 %p6, %rd5, %rd2;
-; SM60-NEXT: min.f64 %rd15, %rd5, %rd2;
+; SM60-NEXT: mov.b64 %rd6, %rd27;
+; SM60-NEXT: setp.eq.b64 %p5, %rd6, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd14, %rd6, %rd4, %p5;
+; SM60-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
+; SM60-NEXT: min.f64 %rd15, %rd6, %rd4;
; SM60-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
-; SM60-NEXT: setp.eq.f64 %p7, %rd5, %rd2;
+; SM60-NEXT: setp.eq.f64 %p7, %rd6, %rd4;
; SM60-NEXT: selp.f64 %rd17, %rd14, %rd16, %p7;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd9+8], %rd5, %rd17;
-; SM60-NEXT: setp.ne.b64 %p8, %rd27, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd9+8], %rd6, %rd17;
+; SM60-NEXT: setp.ne.b64 %p8, %rd27, %rd6;
; SM60-NEXT: @%p8 bra $L__BB282_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end7
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd9+16];
-; SM60-NEXT: $L__BB282_5: // %atomicrmw.start19
+; SM60-NEXT: $L__BB282_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd7, %rd28;
-; SM60-NEXT: setp.eq.b64 %p9, %rd7, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd18, %rd7, %rd1, %p9;
-; SM60-NEXT: setp.nan.f64 %p10, %rd7, %rd1;
-; SM60-NEXT: min.f64 %rd19, %rd7, %rd1;
+; SM60-NEXT: setp.eq.b64 %p9, %rd28, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd18, %rd28, %rd1, %p9;
+; SM60-NEXT: setp.nan.f64 %p10, %rd28, %rd1;
+; SM60-NEXT: min.f64 %rd19, %rd28, %rd1;
; SM60-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p10;
-; SM60-NEXT: setp.eq.f64 %p11, %rd7, %rd1;
+; SM60-NEXT: setp.eq.f64 %p11, %rd28, %rd1;
; SM60-NEXT: selp.f64 %rd21, %rd18, %rd20, %p11;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd9+16], %rd7, %rd21;
-; SM60-NEXT: setp.ne.b64 %p12, %rd28, %rd7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd28, %rd21;
+; SM60-NEXT: setp.ne.b64 %p12, %rd7, %rd28;
+; SM60-NEXT: mov.b64 %rd28, %rd7;
; SM60-NEXT: @%p12 bra $L__BB282_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end18
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd9+24];
-; SM60-NEXT: $L__BB282_7: // %atomicrmw.start26
+; SM60-NEXT: $L__BB282_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.eq.b64 %p13, %rd29, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd22, %rd29, %rd6, %p13;
-; SM60-NEXT: setp.nan.f64 %p14, %rd29, %rd6;
-; SM60-NEXT: min.f64 %rd23, %rd29, %rd6;
+; SM60-NEXT: selp.f64 %rd22, %rd29, %rd2, %p13;
+; SM60-NEXT: setp.nan.f64 %p14, %rd29, %rd2;
+; SM60-NEXT: min.f64 %rd23, %rd29, %rd2;
; SM60-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p14;
-; SM60-NEXT: setp.eq.f64 %p15, %rd29, %rd6;
+; SM60-NEXT: setp.eq.f64 %p15, %rd29, %rd2;
; SM60-NEXT: selp.f64 %rd25, %rd22, %rd24, %p15;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd29, %rd25;
; SM60-NEXT: setp.ne.b64 %p16, %rd8, %rd29;
; SM60-NEXT: mov.b64 %rd29, %rd8;
; SM60-NEXT: @%p16 bra $L__BB282_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
@@ -15810,132 +15840,132 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [fminimum_acq_rel_v8f64_global_cta_param_1+48];
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [fminimum_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fminimum_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd50, [%rd17];
; SM60-NEXT: $L__BB283_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd50;
-; SM60-NEXT: setp.eq.b64 %p1, %rd5, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd18, %rd5, %rd4, %p1;
-; SM60-NEXT: setp.nan.f64 %p2, %rd5, %rd4;
-; SM60-NEXT: min.f64 %rd19, %rd5, %rd4;
+; SM60-NEXT: mov.b64 %rd7, %rd50;
+; SM60-NEXT: setp.eq.b64 %p1, %rd7, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd18, %rd7, %rd5, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd7, %rd5;
+; SM60-NEXT: min.f64 %rd19, %rd7, %rd5;
; SM60-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p2;
-; SM60-NEXT: setp.eq.f64 %p3, %rd5, %rd4;
+; SM60-NEXT: setp.eq.f64 %p3, %rd7, %rd5;
; SM60-NEXT: selp.f64 %rd21, %rd18, %rd20, %p3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd50, [%rd17], %rd5, %rd21;
-; SM60-NEXT: setp.ne.b64 %p4, %rd50, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd50, [%rd17], %rd7, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd50, %rd7;
; SM60-NEXT: @%p4 bra $L__BB283_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd51, [%rd17+8];
-; SM60-NEXT: $L__BB283_3: // %atomicrmw.start12
+; SM60-NEXT: $L__BB283_3: // %atomicrmw.start10
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd6, %rd51;
-; SM60-NEXT: setp.eq.b64 %p5, %rd6, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd22, %rd6, %rd3, %p5;
-; SM60-NEXT: setp.nan.f64 %p6, %rd6, %rd3;
-; SM60-NEXT: min.f64 %rd23, %rd6, %rd3;
+; SM60-NEXT: mov.b64 %rd8, %rd51;
+; SM60-NEXT: setp.eq.b64 %p5, %rd8, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd22, %rd8, %rd6, %p5;
+; SM60-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
+; SM60-NEXT: min.f64 %rd23, %rd8, %rd6;
; SM60-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
-; SM60-NEXT: setp.eq.f64 %p7, %rd6, %rd3;
+; SM60-NEXT: setp.eq.f64 %p7, %rd8, %rd6;
; SM60-NEXT: selp.f64 %rd25, %rd22, %rd24, %p7;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd51, [%rd17+8], %rd6, %rd25;
-; SM60-NEXT: setp.ne.b64 %p8, %rd51, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd51, [%rd17+8], %rd8, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd51, %rd8;
; SM60-NEXT: @%p8 bra $L__BB283_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end11
+; SM60-NEXT: // %bb.4: // %atomicrmw.end9
; SM60-NEXT: ld.volatile.global.b64 %rd52, [%rd17+16];
-; SM60-NEXT: $L__BB283_5: // %atomicrmw.start23
+; SM60-NEXT: $L__BB283_5: // %atomicrmw.start19
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd8, %rd52;
-; SM60-NEXT: setp.eq.b64 %p9, %rd8, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd26, %rd8, %rd2, %p9;
-; SM60-NEXT: setp.nan.f64 %p10, %rd8, %rd2;
-; SM60-NEXT: min.f64 %rd27, %rd8, %rd2;
+; SM60-NEXT: mov.b64 %rd9, %rd52;
+; SM60-NEXT: setp.eq.b64 %p9, %rd9, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd26, %rd9, %rd3, %p9;
+; SM60-NEXT: setp.nan.f64 %p10, %rd9, %rd3;
+; SM60-NEXT: min.f64 %rd27, %rd9, %rd3;
; SM60-NEXT: selp.f64 %rd28, 0d7FF8000000000000, %rd27, %p10;
-; SM60-NEXT: setp.eq.f64 %p11, %rd8, %rd2;
+; SM60-NEXT: setp.eq.f64 %p11, %rd9, %rd3;
; SM60-NEXT: selp.f64 %rd29, %rd26, %rd28, %p11;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd52, [%rd17+16], %rd8, %rd29;
-; SM60-NEXT: setp.ne.b64 %p12, %rd52, %rd8;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd52, [%rd17+16], %rd9, %rd29;
+; SM60-NEXT: setp.ne.b64 %p12, %rd52, %rd9;
; SM60-NEXT: @%p12 bra $L__BB283_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end22
+; SM60-NEXT: // %bb.6: // %atomicrmw.end18
; SM60-NEXT: ld.volatile.global.b64 %rd53, [%rd17+24];
-; SM60-NEXT: $L__BB283_7: // %atomicrmw.start30
+; SM60-NEXT: $L__BB283_7: // %atomicrmw.start24
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd9, %rd53;
-; SM60-NEXT: setp.eq.b64 %p13, %rd9, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd30, %rd9, %rd7, %p13;
-; SM60-NEXT: setp.nan.f64 %p14, %rd9, %rd7;
-; SM60-NEXT: min.f64 %rd31, %rd9, %rd7;
+; SM60-NEXT: mov.b64 %rd10, %rd53;
+; SM60-NEXT: setp.eq.b64 %p13, %rd10, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd30, %rd10, %rd4, %p13;
+; SM60-NEXT: setp.nan.f64 %p14, %rd10, %rd4;
+; SM60-NEXT: min.f64 %rd31, %rd10, %rd4;
; SM60-NEXT: selp.f64 %rd32, 0d7FF8000000000000, %rd31, %p14;
-; SM60-NEXT: setp.eq.f64 %p15, %rd9, %rd7;
+; SM60-NEXT: setp.eq.f64 %p15, %rd10, %rd4;
; SM60-NEXT: selp.f64 %rd33, %rd30, %rd32, %p15;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd53, [%rd17+24], %rd9, %rd33;
-; SM60-NEXT: setp.ne.b64 %p16, %rd53, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd53, [%rd17+24], %rd10, %rd33;
+; SM60-NEXT: setp.ne.b64 %p16, %rd53, %rd10;
; SM60-NEXT: @%p16 bra $L__BB283_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end29
+; SM60-NEXT: // %bb.8: // %atomicrmw.end23
; SM60-NEXT: ld.volatile.global.b64 %rd54, [%rd17+32];
-; SM60-NEXT: $L__BB283_9: // %atomicrmw.start45
+; SM60-NEXT: $L__BB283_9: // %atomicrmw.start37
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd12, %rd54;
-; SM60-NEXT: setp.eq.b64 %p17, %rd12, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd34, %rd12, %rd1, %p17;
-; SM60-NEXT: setp.nan.f64 %p18, %rd12, %rd1;
-; SM60-NEXT: min.f64 %rd35, %rd12, %rd1;
+; SM60-NEXT: mov.b64 %rd13, %rd54;
+; SM60-NEXT: setp.eq.b64 %p17, %rd13, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd34, %rd13, %rd1, %p17;
+; SM60-NEXT: setp.nan.f64 %p18, %rd13, %rd1;
+; SM60-NEXT: min.f64 %rd35, %rd13, %rd1;
; SM60-NEXT: selp.f64 %rd36, 0d7FF8000000000000, %rd35, %p18;
-; SM60-NEXT: setp.eq.f64 %p19, %rd12, %rd1;
+; SM60-NEXT: setp.eq.f64 %p19, %rd13, %rd1;
; SM60-NEXT: selp.f64 %rd37, %rd34, %rd36, %p19;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd54, [%rd17+32], %rd12, %rd37;
-; SM60-NEXT: setp.ne.b64 %p20, %rd54, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd54, [%rd17+32], %rd13, %rd37;
+; SM60-NEXT: setp.ne.b64 %p20, %rd54, %rd13;
; SM60-NEXT: @%p20 bra $L__BB283_9;
-; SM60-NEXT: // %bb.10: // %atomicrmw.end44
+; SM60-NEXT: // %bb.10: // %atomicrmw.end36
; SM60-NEXT: ld.volatile.global.b64 %rd55, [%rd17+40];
-; SM60-NEXT: $L__BB283_11: // %atomicrmw.start52
+; SM60-NEXT: $L__BB283_11: // %atomicrmw.start42
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd13, %rd55;
-; SM60-NEXT: setp.eq.b64 %p21, %rd13, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd38, %rd13, %rd11, %p21;
-; SM60-NEXT: setp.nan.f64 %p22, %rd13, %rd11;
-; SM60-NEXT: min.f64 %rd39, %rd13, %rd11;
+; SM60-NEXT: mov.b64 %rd14, %rd55;
+; SM60-NEXT: setp.eq.b64 %p21, %rd14, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd38, %rd14, %rd2, %p21;
+; SM60-NEXT: setp.nan.f64 %p22, %rd14, %rd2;
+; SM60-NEXT: min.f64 %rd39, %rd14, %rd2;
; SM60-NEXT: selp.f64 %rd40, 0d7FF8000000000000, %rd39, %p22;
-; SM60-NEXT: setp.eq.f64 %p23, %rd13, %rd11;
+; SM60-NEXT: setp.eq.f64 %p23, %rd14, %rd2;
; SM60-NEXT: selp.f64 %rd41, %rd38, %rd40, %p23;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd55, [%rd17+40], %rd13, %rd41;
-; SM60-NEXT: setp.ne.b64 %p24, %rd55, %rd13;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd55, [%rd17+40], %rd14, %rd41;
+; SM60-NEXT: setp.ne.b64 %p24, %rd55, %rd14;
; SM60-NEXT: @%p24 bra $L__BB283_11;
-; SM60-NEXT: // %bb.12: // %atomicrmw.end51
+; SM60-NEXT: // %bb.12: // %atomicrmw.end41
; SM60-NEXT: ld.volatile.global.b64 %rd56, [%rd17+48];
-; SM60-NEXT: $L__BB283_13: // %atomicrmw.start63
+; SM60-NEXT: $L__BB283_13: // %atomicrmw.start51
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd15, %rd56;
-; SM60-NEXT: setp.eq.b64 %p25, %rd15, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd42, %rd15, %rd10, %p25;
-; SM60-NEXT: setp.nan.f64 %p26, %rd15, %rd10;
-; SM60-NEXT: min.f64 %rd43, %rd15, %rd10;
+; SM60-NEXT: setp.eq.b64 %p25, %rd56, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd42, %rd56, %rd11, %p25;
+; SM60-NEXT: setp.nan.f64 %p26, %rd56, %rd11;
+; SM60-NEXT: min.f64 %rd43, %rd56, %rd11;
; SM60-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
-; SM60-NEXT: setp.eq.f64 %p27, %rd15, %rd10;
+; SM60-NEXT: setp.eq.f64 %p27, %rd56, %rd11;
; SM60-NEXT: selp.f64 %rd45, %rd42, %rd44, %p27;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd56, [%rd17+48], %rd15, %rd45;
-; SM60-NEXT: setp.ne.b64 %p28, %rd56, %rd15;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd56, %rd45;
+; SM60-NEXT: setp.ne.b64 %p28, %rd15, %rd56;
+; SM60-NEXT: mov.b64 %rd56, %rd15;
; SM60-NEXT: @%p28 bra $L__BB283_13;
-; SM60-NEXT: // %bb.14: // %atomicrmw.end62
+; SM60-NEXT: // %bb.14: // %atomicrmw.end50
; SM60-NEXT: ld.volatile.global.b64 %rd57, [%rd17+56];
-; SM60-NEXT: $L__BB283_15: // %atomicrmw.start70
+; SM60-NEXT: $L__BB283_15: // %atomicrmw.start56
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.eq.b64 %p29, %rd57, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd46, %rd57, %rd14, %p29;
-; SM60-NEXT: setp.nan.f64 %p30, %rd57, %rd14;
-; SM60-NEXT: min.f64 %rd47, %rd57, %rd14;
+; SM60-NEXT: selp.f64 %rd46, %rd57, %rd12, %p29;
+; SM60-NEXT: setp.nan.f64 %p30, %rd57, %rd12;
+; SM60-NEXT: min.f64 %rd47, %rd57, %rd12;
; SM60-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p30;
-; SM60-NEXT: setp.eq.f64 %p31, %rd57, %rd14;
+; SM60-NEXT: setp.eq.f64 %p31, %rd57, %rd12;
; SM60-NEXT: selp.f64 %rd49, %rd46, %rd48, %p31;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd57, %rd49;
; SM60-NEXT: setp.ne.b64 %p32, %rd16, %rd57;
; SM60-NEXT: mov.b64 %rd57, %rd16;
; SM60-NEXT: @%p32 bra $L__BB283_15;
-; SM60-NEXT: // %bb.16: // %atomicrmw.end69
+; SM60-NEXT: // %bb.16: // %atomicrmw.end55
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd56, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd54, %rd55};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
@@ -15951,26 +15981,26 @@ define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: .reg .b64 %rd<9>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd3];
+; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd2];
; SM60-NEXT: $L__BB284_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.eq.b64 %p1, %rd8, 0;
-; SM60-NEXT: selp.f64 %rd4, %rd8, %rd1, %p1;
-; SM60-NEXT: setp.nan.f64 %p2, %rd8, %rd1;
-; SM60-NEXT: max.f64 %rd5, %rd8, %rd1;
+; SM60-NEXT: selp.f64 %rd4, %rd8, %rd3, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd8, %rd3;
+; SM60-NEXT: max.f64 %rd5, %rd8, %rd3;
; SM60-NEXT: selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
-; SM60-NEXT: setp.eq.f64 %p3, %rd8, %rd1;
+; SM60-NEXT: setp.eq.f64 %p3, %rd8, %rd3;
; SM60-NEXT: selp.f64 %rd7, %rd4, %rd6, %p3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
-; SM60-NEXT: setp.ne.b64 %p4, %rd2, %rd8;
-; SM60-NEXT: mov.b64 %rd8, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd1, [%rd2], %rd8, %rd7;
+; SM60-NEXT: setp.ne.b64 %p4, %rd1, %rd8;
+; SM60-NEXT: mov.b64 %rd8, %rd1;
; SM60-NEXT: @%p4 bra $L__BB284_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -15985,39 +16015,39 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd5, [fmaximum_acq_rel_v2f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd5];
; SM60-NEXT: $L__BB285_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd3, %rd14;
-; SM60-NEXT: setp.eq.b64 %p1, %rd3, 0;
-; SM60-NEXT: selp.f64 %rd6, %rd3, %rd2, %p1;
-; SM60-NEXT: setp.nan.f64 %p2, %rd3, %rd2;
-; SM60-NEXT: max.f64 %rd7, %rd3, %rd2;
+; SM60-NEXT: setp.eq.b64 %p1, %rd14, 0;
+; SM60-NEXT: selp.f64 %rd6, %rd14, %rd1, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd14, %rd1;
+; SM60-NEXT: max.f64 %rd7, %rd14, %rd1;
; SM60-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p2;
-; SM60-NEXT: setp.eq.f64 %p3, %rd3, %rd2;
+; SM60-NEXT: setp.eq.f64 %p3, %rd14, %rd1;
; SM60-NEXT: selp.f64 %rd9, %rd6, %rd8, %p3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd5], %rd3, %rd9;
-; SM60-NEXT: setp.ne.b64 %p4, %rd14, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd14, %rd9;
+; SM60-NEXT: setp.ne.b64 %p4, %rd3, %rd14;
+; SM60-NEXT: mov.b64 %rd14, %rd3;
; SM60-NEXT: @%p4 bra $L__BB285_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd5+8];
-; SM60-NEXT: $L__BB285_3: // %atomicrmw.start4
+; SM60-NEXT: $L__BB285_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.eq.b64 %p5, %rd15, 0;
-; SM60-NEXT: selp.f64 %rd10, %rd15, %rd1, %p5;
-; SM60-NEXT: setp.nan.f64 %p6, %rd15, %rd1;
-; SM60-NEXT: max.f64 %rd11, %rd15, %rd1;
+; SM60-NEXT: selp.f64 %rd10, %rd15, %rd2, %p5;
+; SM60-NEXT: setp.nan.f64 %p6, %rd15, %rd2;
+; SM60-NEXT: max.f64 %rd11, %rd15, %rd2;
; SM60-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
-; SM60-NEXT: setp.eq.f64 %p7, %rd15, %rd1;
+; SM60-NEXT: setp.eq.f64 %p7, %rd15, %rd2;
; SM60-NEXT: selp.f64 %rd13, %rd10, %rd12, %p7;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd15, %rd13;
; SM60-NEXT: setp.ne.b64 %p8, %rd4, %rd15;
; SM60-NEXT: mov.b64 %rd15, %rd4;
; SM60-NEXT: @%p8 bra $L__BB285_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end3
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd4};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -16032,70 +16062,70 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd9];
; SM60-NEXT: $L__BB286_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd4, %rd26;
-; SM60-NEXT: setp.eq.b64 %p1, %rd4, 0;
-; SM60-NEXT: selp.f64 %rd10, %rd4, %rd3, %p1;
-; SM60-NEXT: setp.nan.f64 %p2, %rd4, %rd3;
-; SM60-NEXT: max.f64 %rd11, %rd4, %rd3;
+; SM60-NEXT: mov.b64 %rd5, %rd26;
+; SM60-NEXT: setp.eq.b64 %p1, %rd5, 0;
+; SM60-NEXT: selp.f64 %rd10, %rd5, %rd3, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd5, %rd3;
+; SM60-NEXT: max.f64 %rd11, %rd5, %rd3;
; SM60-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p2;
-; SM60-NEXT: setp.eq.f64 %p3, %rd4, %rd3;
+; SM60-NEXT: setp.eq.f64 %p3, %rd5, %rd3;
; SM60-NEXT: selp.f64 %rd13, %rd10, %rd12, %p3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd9], %rd4, %rd13;
-; SM60-NEXT: setp.ne.b64 %p4, %rd26, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd9], %rd5, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd26, %rd5;
; SM60-NEXT: @%p4 bra $L__BB286_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd9+8];
-; SM60-NEXT: $L__BB286_3: // %atomicrmw.start8
+; SM60-NEXT: $L__BB286_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd27;
-; SM60-NEXT: setp.eq.b64 %p5, %rd5, 0;
-; SM60-NEXT: selp.f64 %rd14, %rd5, %rd2, %p5;
-; SM60-NEXT: setp.nan.f64 %p6, %rd5, %rd2;
-; SM60-NEXT: max.f64 %rd15, %rd5, %rd2;
+; SM60-NEXT: mov.b64 %rd6, %rd27;
+; SM60-NEXT: setp.eq.b64 %p5, %rd6, 0;
+; SM60-NEXT: selp.f64 %rd14, %rd6, %rd4, %p5;
+; SM60-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
+; SM60-NEXT: max.f64 %rd15, %rd6, %rd4;
; SM60-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
-; SM60-NEXT: setp.eq.f64 %p7, %rd5, %rd2;
+; SM60-NEXT: setp.eq.f64 %p7, %rd6, %rd4;
; SM60-NEXT: selp.f64 %rd17, %rd14, %rd16, %p7;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd9+8], %rd5, %rd17;
-; SM60-NEXT: setp.ne.b64 %p8, %rd27, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd9+8], %rd6, %rd17;
+; SM60-NEXT: setp.ne.b64 %p8, %rd27, %rd6;
; SM60-NEXT: @%p8 bra $L__BB286_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end7
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd9+16];
-; SM60-NEXT: $L__BB286_5: // %atomicrmw.start19
+; SM60-NEXT: $L__BB286_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd7, %rd28;
-; SM60-NEXT: setp.eq.b64 %p9, %rd7, 0;
-; SM60-NEXT: selp.f64 %rd18, %rd7, %rd1, %p9;
-; SM60-NEXT: setp.nan.f64 %p10, %rd7, %rd1;
-; SM60-NEXT: max.f64 %rd19, %rd7, %rd1;
+; SM60-NEXT: setp.eq.b64 %p9, %rd28, 0;
+; SM60-NEXT: selp.f64 %rd18, %rd28, %rd1, %p9;
+; SM60-NEXT: setp.nan.f64 %p10, %rd28, %rd1;
+; SM60-NEXT: max.f64 %rd19, %rd28, %rd1;
; SM60-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p10;
-; SM60-NEXT: setp.eq.f64 %p11, %rd7, %rd1;
+; SM60-NEXT: setp.eq.f64 %p11, %rd28, %rd1;
; SM60-NEXT: selp.f64 %rd21, %rd18, %rd20, %p11;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd9+16], %rd7, %rd21;
-; SM60-NEXT: setp.ne.b64 %p12, %rd28, %rd7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd28, %rd21;
+; SM60-NEXT: setp.ne.b64 %p12, %rd7, %rd28;
+; SM60-NEXT: mov.b64 %rd28, %rd7;
; SM60-NEXT: @%p12 bra $L__BB286_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end18
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd9+24];
-; SM60-NEXT: $L__BB286_7: // %atomicrmw.start26
+; SM60-NEXT: $L__BB286_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.eq.b64 %p13, %rd29, 0;
-; SM60-NEXT: selp.f64 %rd22, %rd29, %rd6, %p13;
-; SM60-NEXT: setp.nan.f64 %p14, %rd29, %rd6;
-; SM60-NEXT: max.f64 %rd23, %rd29, %rd6;
+; SM60-NEXT: selp.f64 %rd22, %rd29, %rd2, %p13;
+; SM60-NEXT: setp.nan.f64 %p14, %rd29, %rd2;
+; SM60-NEXT: max.f64 %rd23, %rd29, %rd2;
; SM60-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p14;
-; SM60-NEXT: setp.eq.f64 %p15, %rd29, %rd6;
+; SM60-NEXT: setp.eq.f64 %p15, %rd29, %rd2;
; SM60-NEXT: selp.f64 %rd25, %rd22, %rd24, %p15;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd29, %rd25;
; SM60-NEXT: setp.ne.b64 %p16, %rd8, %rd29;
; SM60-NEXT: mov.b64 %rd29, %rd8;
; SM60-NEXT: @%p16 bra $L__BB286_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
@@ -16111,132 +16141,132 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [fmaximum_acq_rel_v8f64_global_cta_param_1+48];
-; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
-; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmaximum_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd50, [%rd17];
; SM60-NEXT: $L__BB287_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd50;
-; SM60-NEXT: setp.eq.b64 %p1, %rd5, 0;
-; SM60-NEXT: selp.f64 %rd18, %rd5, %rd4, %p1;
-; SM60-NEXT: setp.nan.f64 %p2, %rd5, %rd4;
-; SM60-NEXT: max.f64 %rd19, %rd5, %rd4;
+; SM60-NEXT: mov.b64 %rd7, %rd50;
+; SM60-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM60-NEXT: selp.f64 %rd18, %rd7, %rd5, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd7, %rd5;
+; SM60-NEXT: max.f64 %rd19, %rd7, %rd5;
; SM60-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p2;
-; SM60-NEXT: setp.eq.f64 %p3, %rd5, %rd4;
+; SM60-NEXT: setp.eq.f64 %p3, %rd7, %rd5;
; SM60-NEXT: selp.f64 %rd21, %rd18, %rd20, %p3;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd50, [%rd17], %rd5, %rd21;
-; SM60-NEXT: setp.ne.b64 %p4, %rd50, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd50, [%rd17], %rd7, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd50, %rd7;
; SM60-NEXT: @%p4 bra $L__BB287_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd51, [%rd17+8];
-; SM60-NEXT: $L__BB287_3: // %atomicrmw.start12
+; SM60-NEXT: $L__BB287_3: // %atomicrmw.start10
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd6, %rd51;
-; SM60-NEXT: setp.eq.b64 %p5, %rd6, 0;
-; SM60-NEXT: selp.f64 %rd22, %rd6, %rd3, %p5;
-; SM60-NEXT: setp.nan.f64 %p6, %rd6, %rd3;
-; SM60-NEXT: max.f64 %rd23, %rd6, %rd3;
+; SM60-NEXT: mov.b64 %rd8, %rd51;
+; SM60-NEXT: setp.eq.b64 %p5, %rd8, 0;
+; SM60-NEXT: selp.f64 %rd22, %rd8, %rd6, %p5;
+; SM60-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
+; SM60-NEXT: max.f64 %rd23, %rd8, %rd6;
; SM60-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
-; SM60-NEXT: setp.eq.f64 %p7, %rd6, %rd3;
+; SM60-NEXT: setp.eq.f64 %p7, %rd8, %rd6;
; SM60-NEXT: selp.f64 %rd25, %rd22, %rd24, %p7;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd51, [%rd17+8], %rd6, %rd25;
-; SM60-NEXT: setp.ne.b64 %p8, %rd51, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd51, [%rd17+8], %rd8, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd51, %rd8;
; SM60-NEXT: @%p8 bra $L__BB287_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end11
+; SM60-NEXT: // %bb.4: // %atomicrmw.end9
; SM60-NEXT: ld.volatile.global.b64 %rd52, [%rd17+16];
-; SM60-NEXT: $L__BB287_5: // %atomicrmw.start23
+; SM60-NEXT: $L__BB287_5: // %atomicrmw.start19
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd8, %rd52;
-; SM60-NEXT: setp.eq.b64 %p9, %rd8, 0;
-; SM60-NEXT: selp.f64 %rd26, %rd8, %rd2, %p9;
-; SM60-NEXT: setp.nan.f64 %p10, %rd8, %rd2;
-; SM60-NEXT: max.f64 %rd27, %rd8, %rd2;
+; SM60-NEXT: mov.b64 %rd9, %rd52;
+; SM60-NEXT: setp.eq.b64 %p9, %rd9, 0;
+; SM60-NEXT: selp.f64 %rd26, %rd9, %rd3, %p9;
+; SM60-NEXT: setp.nan.f64 %p10, %rd9, %rd3;
+; SM60-NEXT: max.f64 %rd27, %rd9, %rd3;
; SM60-NEXT: selp.f64 %rd28, 0d7FF8000000000000, %rd27, %p10;
-; SM60-NEXT: setp.eq.f64 %p11, %rd8, %rd2;
+; SM60-NEXT: setp.eq.f64 %p11, %rd9, %rd3;
; SM60-NEXT: selp.f64 %rd29, %rd26, %rd28, %p11;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd52, [%rd17+16], %rd8, %rd29;
-; SM60-NEXT: setp.ne.b64 %p12, %rd52, %rd8;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd52, [%rd17+16], %rd9, %rd29;
+; SM60-NEXT: setp.ne.b64 %p12, %rd52, %rd9;
; SM60-NEXT: @%p12 bra $L__BB287_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end22
+; SM60-NEXT: // %bb.6: // %atomicrmw.end18
; SM60-NEXT: ld.volatile.global.b64 %rd53, [%rd17+24];
-; SM60-NEXT: $L__BB287_7: // %atomicrmw.start30
+; SM60-NEXT: $L__BB287_7: // %atomicrmw.start24
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd9, %rd53;
-; SM60-NEXT: setp.eq.b64 %p13, %rd9, 0;
-; SM60-NEXT: selp.f64 %rd30, %rd9, %rd7, %p13;
-; SM60-NEXT: setp.nan.f64 %p14, %rd9, %rd7;
-; SM60-NEXT: max.f64 %rd31, %rd9, %rd7;
+; SM60-NEXT: mov.b64 %rd10, %rd53;
+; SM60-NEXT: setp.eq.b64 %p13, %rd10, 0;
+; SM60-NEXT: selp.f64 %rd30, %rd10, %rd4, %p13;
+; SM60-NEXT: setp.nan.f64 %p14, %rd10, %rd4;
+; SM60-NEXT: max.f64 %rd31, %rd10, %rd4;
; SM60-NEXT: selp.f64 %rd32, 0d7FF8000000000000, %rd31, %p14;
-; SM60-NEXT: setp.eq.f64 %p15, %rd9, %rd7;
+; SM60-NEXT: setp.eq.f64 %p15, %rd10, %rd4;
; SM60-NEXT: selp.f64 %rd33, %rd30, %rd32, %p15;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd53, [%rd17+24], %rd9, %rd33;
-; SM60-NEXT: setp.ne.b64 %p16, %rd53, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd53, [%rd17+24], %rd10, %rd33;
+; SM60-NEXT: setp.ne.b64 %p16, %rd53, %rd10;
; SM60-NEXT: @%p16 bra $L__BB287_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end29
+; SM60-NEXT: // %bb.8: // %atomicrmw.end23
; SM60-NEXT: ld.volatile.global.b64 %rd54, [%rd17+32];
-; SM60-NEXT: $L__BB287_9: // %atomicrmw.start45
+; SM60-NEXT: $L__BB287_9: // %atomicrmw.start37
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd12, %rd54;
-; SM60-NEXT: setp.eq.b64 %p17, %rd12, 0;
-; SM60-NEXT: selp.f64 %rd34, %rd12, %rd1, %p17;
-; SM60-NEXT: setp.nan.f64 %p18, %rd12, %rd1;
-; SM60-NEXT: max.f64 %rd35, %rd12, %rd1;
+; SM60-NEXT: mov.b64 %rd13, %rd54;
+; SM60-NEXT: setp.eq.b64 %p17, %rd13, 0;
+; SM60-NEXT: selp.f64 %rd34, %rd13, %rd1, %p17;
+; SM60-NEXT: setp.nan.f64 %p18, %rd13, %rd1;
+; SM60-NEXT: max.f64 %rd35, %rd13, %rd1;
; SM60-NEXT: selp.f64 %rd36, 0d7FF8000000000000, %rd35, %p18;
-; SM60-NEXT: setp.eq.f64 %p19, %rd12, %rd1;
+; SM60-NEXT: setp.eq.f64 %p19, %rd13, %rd1;
; SM60-NEXT: selp.f64 %rd37, %rd34, %rd36, %p19;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd54, [%rd17+32], %rd12, %rd37;
-; SM60-NEXT: setp.ne.b64 %p20, %rd54, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd54, [%rd17+32], %rd13, %rd37;
+; SM60-NEXT: setp.ne.b64 %p20, %rd54, %rd13;
; SM60-NEXT: @%p20 bra $L__BB287_9;
-; SM60-NEXT: // %bb.10: // %atomicrmw.end44
+; SM60-NEXT: // %bb.10: // %atomicrmw.end36
; SM60-NEXT: ld.volatile.global.b64 %rd55, [%rd17+40];
-; SM60-NEXT: $L__BB287_11: // %atomicrmw.start52
+; SM60-NEXT: $L__BB287_11: // %atomicrmw.start42
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd13, %rd55;
-; SM60-NEXT: setp.eq.b64 %p21, %rd13, 0;
-; SM60-NEXT: selp.f64 %rd38, %rd13, %rd11, %p21;
-; SM60-NEXT: setp.nan.f64 %p22, %rd13, %rd11;
-; SM60-NEXT: max.f64 %rd39, %rd13, %rd11;
+; SM60-NEXT: mov.b64 %rd14, %rd55;
+; SM60-NEXT: setp.eq.b64 %p21, %rd14, 0;
+; SM60-NEXT: selp.f64 %rd38, %rd14, %rd2, %p21;
+; SM60-NEXT: setp.nan.f64 %p22, %rd14, %rd2;
+; SM60-NEXT: max.f64 %rd39, %rd14, %rd2;
; SM60-NEXT: selp.f64 %rd40, 0d7FF8000000000000, %rd39, %p22;
-; SM60-NEXT: setp.eq.f64 %p23, %rd13, %rd11;
+; SM60-NEXT: setp.eq.f64 %p23, %rd14, %rd2;
; SM60-NEXT: selp.f64 %rd41, %rd38, %rd40, %p23;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd55, [%rd17+40], %rd13, %rd41;
-; SM60-NEXT: setp.ne.b64 %p24, %rd55, %rd13;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd55, [%rd17+40], %rd14, %rd41;
+; SM60-NEXT: setp.ne.b64 %p24, %rd55, %rd14;
; SM60-NEXT: @%p24 bra $L__BB287_11;
-; SM60-NEXT: // %bb.12: // %atomicrmw.end51
+; SM60-NEXT: // %bb.12: // %atomicrmw.end41
; SM60-NEXT: ld.volatile.global.b64 %rd56, [%rd17+48];
-; SM60-NEXT: $L__BB287_13: // %atomicrmw.start63
+; SM60-NEXT: $L__BB287_13: // %atomicrmw.start51
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd15, %rd56;
-; SM60-NEXT: setp.eq.b64 %p25, %rd15, 0;
-; SM60-NEXT: selp.f64 %rd42, %rd15, %rd10, %p25;
-; SM60-NEXT: setp.nan.f64 %p26, %rd15, %rd10;
-; SM60-NEXT: max.f64 %rd43, %rd15, %rd10;
+; SM60-NEXT: setp.eq.b64 %p25, %rd56, 0;
+; SM60-NEXT: selp.f64 %rd42, %rd56, %rd11, %p25;
+; SM60-NEXT: setp.nan.f64 %p26, %rd56, %rd11;
+; SM60-NEXT: max.f64 %rd43, %rd56, %rd11;
; SM60-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
-; SM60-NEXT: setp.eq.f64 %p27, %rd15, %rd10;
+; SM60-NEXT: setp.eq.f64 %p27, %rd56, %rd11;
; SM60-NEXT: selp.f64 %rd45, %rd42, %rd44, %p27;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd56, [%rd17+48], %rd15, %rd45;
-; SM60-NEXT: setp.ne.b64 %p28, %rd56, %rd15;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd56, %rd45;
+; SM60-NEXT: setp.ne.b64 %p28, %rd15, %rd56;
+; SM60-NEXT: mov.b64 %rd56, %rd15;
; SM60-NEXT: @%p28 bra $L__BB287_13;
-; SM60-NEXT: // %bb.14: // %atomicrmw.end62
+; SM60-NEXT: // %bb.14: // %atomicrmw.end50
; SM60-NEXT: ld.volatile.global.b64 %rd57, [%rd17+56];
-; SM60-NEXT: $L__BB287_15: // %atomicrmw.start70
+; SM60-NEXT: $L__BB287_15: // %atomicrmw.start56
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.eq.b64 %p29, %rd57, 0;
-; SM60-NEXT: selp.f64 %rd46, %rd57, %rd14, %p29;
-; SM60-NEXT: setp.nan.f64 %p30, %rd57, %rd14;
-; SM60-NEXT: max.f64 %rd47, %rd57, %rd14;
+; SM60-NEXT: selp.f64 %rd46, %rd57, %rd12, %p29;
+; SM60-NEXT: setp.nan.f64 %p30, %rd57, %rd12;
+; SM60-NEXT: max.f64 %rd47, %rd57, %rd12;
; SM60-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p30;
-; SM60-NEXT: setp.eq.f64 %p31, %rd57, %rd14;
+; SM60-NEXT: setp.eq.f64 %p31, %rd57, %rd12;
; SM60-NEXT: selp.f64 %rd49, %rd46, %rd48, %p31;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd57, %rd49;
; SM60-NEXT: setp.ne.b64 %p32, %rd16, %rd57;
; SM60-NEXT: mov.b64 %rd57, %rd16;
; SM60-NEXT: @%p32 bra $L__BB287_15;
-; SM60-NEXT: // %bb.16: // %atomicrmw.end69
+; SM60-NEXT: // %bb.16: // %atomicrmw.end55
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd56, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd54, %rd55};
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
@@ -16254,9 +16284,9 @@ define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f16_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -16291,9 +16321,9 @@ define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f16_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -16328,9 +16358,9 @@ define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f16_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -16365,9 +16395,9 @@ define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f16_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -16874,9 +16904,9 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1f16_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -16911,9 +16941,9 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1f16_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -16948,9 +16978,9 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1f16_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -16985,9 +17015,9 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1f16_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -17261,225 +17291,241 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM60-NOFTZ-DISALLOW: {
; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
-; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<17>;
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<22>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f16_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: xor.b32 %r2, %r6, -2147450880;
+; SM60-NOFTZ-DISALLOW-NEXT: xor.b32 %r1, %r5, -2147450880;
+; SM60-NOFTZ-DISALLOW-NEXT: xor.b32 %r4, %r8, -2147450880;
+; SM60-NOFTZ-DISALLOW-NEXT: xor.b32 %r3, %r7, -2147450880;
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r13, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB295_1: // %atomicrmw.start
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f16x2 %r5, %r9, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f16x2 %r6, %r10, %r2;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f16x2 %r9, %r13, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f16x2 %r10, %r14, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r10;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r9;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r14;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r13, %rd11;
; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB295_1;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r15, %rd12;
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB295_3: // %atomicrmw.start2
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f16x2 %r7, %r11, %r3;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r7;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f16x2 %r8, %r12, %r4;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f16x2 %r11, %r15, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f16x2 %r12, %r16, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r12;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r16;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r15, %rd21;
; SM60-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB295_3;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
; SM60-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM60-NOFTZ-ALLOW: {
; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<3>;
-; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<13>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<17>;
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<22>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f16_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: xor.b32 %r2, %r6, -2147450880;
+; SM60-NOFTZ-ALLOW-NEXT: xor.b32 %r1, %r5, -2147450880;
+; SM60-NOFTZ-ALLOW-NEXT: xor.b32 %r4, %r8, -2147450880;
+; SM60-NOFTZ-ALLOW-NEXT: xor.b32 %r3, %r7, -2147450880;
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r13, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: $L__BB295_1: // %atomicrmw.start
; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f16x2 %r5, %r9, %r1;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f16x2 %r6, %r10, %r2;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f16x2 %r9, %r13, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f16x2 %r10, %r14, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r10;
; SM60-NOFTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NOFTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r9;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r14;
; SM60-NOFTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NOFTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r13, %rd11;
; SM60-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB295_1;
; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r15, %rd12;
; SM60-NOFTZ-ALLOW-NEXT: $L__BB295_3: // %atomicrmw.start2
; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f16x2 %r7, %r11, %r3;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r7;
-; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f16x2 %r8, %r12, %r4;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f16x2 %r11, %r15, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f16x2 %r12, %r16, %r4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r12;
; SM60-NOFTZ-ALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NOFTZ-ALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r16;
; SM60-NOFTZ-ALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NOFTZ-ALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
-; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r15, %rd21;
; SM60-NOFTZ-ALLOW-NEXT: @%p2 bra $L__BB295_3;
; SM60-NOFTZ-ALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
; SM60-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM60-FTZ-DISALLOW: {
; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<3>;
-; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<17>;
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<22>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f16_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: xor.b32 %r2, %r6, -2147450880;
+; SM60-FTZ-DISALLOW-NEXT: xor.b32 %r1, %r5, -2147450880;
+; SM60-FTZ-DISALLOW-NEXT: xor.b32 %r4, %r8, -2147450880;
+; SM60-FTZ-DISALLOW-NEXT: xor.b32 %r3, %r7, -2147450880;
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r13, %rd2;
; SM60-FTZ-DISALLOW-NEXT: $L__BB295_1: // %atomicrmw.start
; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f16x2 %r5, %r9, %r1;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f16x2 %r6, %r10, %r2;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f16x2 %r9, %r13, %r1;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f16x2 %r10, %r14, %r2;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r10;
; SM60-FTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-FTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r9;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r14;
; SM60-FTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-FTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM60-FTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r13, %rd11;
; SM60-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB295_1;
; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r15, %rd12;
; SM60-FTZ-DISALLOW-NEXT: $L__BB295_3: // %atomicrmw.start2
; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f16x2 %r7, %r11, %r3;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r7;
-; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f16x2 %r8, %r12, %r4;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f16x2 %r11, %r15, %r3;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f16x2 %r12, %r16, %r4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r12;
; SM60-FTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-FTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r16;
; SM60-FTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-FTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM60-FTZ-DISALLOW-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
-; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r15, %rd21;
; SM60-FTZ-DISALLOW-NEXT: @%p2 bra $L__BB295_3;
; SM60-FTZ-DISALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM60-FTZ-DISALLOW-NEXT: ret;
;
; SM60-FTZ-ALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM60-FTZ-ALLOW: {
; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<3>;
-; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<13>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<17>;
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<22>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f16_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: xor.b32 %r2, %r6, -2147450880;
+; SM60-FTZ-ALLOW-NEXT: xor.b32 %r1, %r5, -2147450880;
+; SM60-FTZ-ALLOW-NEXT: xor.b32 %r4, %r8, -2147450880;
+; SM60-FTZ-ALLOW-NEXT: xor.b32 %r3, %r7, -2147450880;
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r13, %rd2;
; SM60-FTZ-ALLOW-NEXT: $L__BB295_1: // %atomicrmw.start
; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f16x2 %r5, %r9, %r1;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f16x2 %r6, %r10, %r2;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f16x2 %r9, %r13, %r1;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f16x2 %r10, %r14, %r2;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r10;
; SM60-FTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-FTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r9;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r14;
; SM60-FTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-FTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM60-FTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r13, %rd11;
; SM60-FTZ-ALLOW-NEXT: @%p1 bra $L__BB295_1;
; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r15, %rd12;
; SM60-FTZ-ALLOW-NEXT: $L__BB295_3: // %atomicrmw.start2
; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f16x2 %r7, %r11, %r3;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r7;
-; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f16x2 %r8, %r12, %r4;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f16x2 %r11, %r15, %r3;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f16x2 %r12, %r16, %r4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r12;
; SM60-FTZ-ALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-FTZ-ALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r16;
; SM60-FTZ-ALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-FTZ-ALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM60-FTZ-ALLOW-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
-; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r15, %rd21;
; SM60-FTZ-ALLOW-NEXT: @%p2 bra $L__BB295_3;
; SM60-FTZ-ALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
ret <8 x half> %retval
@@ -17494,9 +17540,9 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -17534,9 +17580,9 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -17574,9 +17620,9 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -17614,9 +17660,9 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -18418,9 +18464,9 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -18458,9 +18504,9 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -18498,9 +18544,9 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -18538,9 +18584,9 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -19342,9 +19388,9 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -19386,9 +19432,9 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -19430,9 +19476,9 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -19474,9 +19520,9 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -20506,9 +20552,9 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -20550,9 +20596,9 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -20594,9 +20640,9 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -20638,9 +20684,9 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -21670,9 +21716,9 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -21715,9 +21761,9 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -21760,9 +21806,9 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -21805,9 +21851,9 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -23006,9 +23052,9 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -23051,9 +23097,9 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -23096,9 +23142,9 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -23141,9 +23187,9 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -23722,612 +23768,628 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NOFTZ-DISALLOW: {
; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<11>;
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<17>;
-; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<93>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<97>;
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<22>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: xor.b32 %r2, %r6, -2147450880;
+; SM60-NOFTZ-DISALLOW-NEXT: xor.b32 %r1, %r5, -2147450880;
+; SM60-NOFTZ-DISALLOW-NEXT: xor.b32 %r4, %r8, -2147450880;
+; SM60-NOFTZ-DISALLOW-NEXT: xor.b32 %r3, %r7, -2147450880;
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r94}, %rd2; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r93, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB319_1: // %atomicrmw.start
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r6, %r5, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r89;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r7, %rs4;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r8, %r7, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r9, %r8, %r6;
-; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r10, %r9, 16, 1;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r11, %r10, %r9;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r12, %r11, 32767;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r9, %r9;
-; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r13, %r9, 4194304;
-; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r14, %r13, %r12, %p1;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r15, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r16, %r15, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r17, %rs3;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r18, %r17, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r19, %r18, %r16;
-; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r21, %r20, %r19;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r22, %r21, 32767;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r23, %r19, 4194304;
-; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; SM60-NOFTZ-DISALLOW-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r25;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r26, %rs6;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r27, %r26, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r90;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r28, %rs8;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r29, %r28, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r30, %r29, %r27;
-; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r31, %r30, 16, 1;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r32, %r31, %r30;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r33, %r32, 32767;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p3, %r30, %r30;
-; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r34, %r30, 4194304;
-; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r35, %r34, %r33, %p3;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r36, %rs5;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r37, %r36, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r38, %rs7;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r39, %r38, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r40, %r39, %r37;
-; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r41, %r40, 16, 1;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r42, %r41, %r40;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r43, %r42, 32767;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p4, %r40, %r40;
-; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r44, %r40, 4194304;
-; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r45, %r44, %r43, %p4;
-; SM60-NOFTZ-DISALLOW-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs2;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r93;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r11, %rs4;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r13, %r12, %r10;
+; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r14, %r13, 16, 1;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, %r13;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r16, %r15, 32767;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r13, %r13;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r17, %r13, 4194304;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r19, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r20, %r19, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r21, %rs3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r22, %r21, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r23, %r22, %r20;
+; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r24, %r23, 16, 1;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r25, %r24, %r23;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r26, %r25, 32767;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r23, %r23;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r27, %r23, 4194304;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r28, %r27, %r26, %p2;
+; SM60-NOFTZ-DISALLOW-NEXT: prmt.b32 %r29, %r28, %r18, 0x7632U;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r29;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r30, %rs6;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r31, %r30, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r94;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r32, %rs8;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r33, %r32, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r34, %r33, %r31;
+; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r35, %r34, 16, 1;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r36, %r35, %r34;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r37, %r36, 32767;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p3, %r34, %r34;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r38, %r34, 4194304;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r39, %r38, %r37, %p3;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r40, %rs5;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r41, %r40, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r42, %rs7;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r43, %r42, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r44, %r43, %r41;
+; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r45, %r44, 16, 1;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r46, %r45, %r44;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r47, %r46, 32767;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p4, %r44, %r44;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r48, %r44, 4194304;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r49, %r48, %r47, %p4;
+; SM60-NOFTZ-DISALLOW-NEXT: prmt.b32 %r50, %r49, %r39, 0x7632U;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r50;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r89;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r93;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r94;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r94}, %rd11; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r93, %rd11;
; SM60-NOFTZ-DISALLOW-NEXT: @%p5 bra $L__BB319_1;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r96}, %rd12; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r95, %rd12;
; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r4;
; SM60-NOFTZ-DISALLOW-NEXT: $L__BB319_3: // %atomicrmw.start2
; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r47, %rs10;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r48, %r47, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r91;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r49, %rs12;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r50, %r49, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r51, %r50, %r48;
-; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r52, %r51, 16, 1;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r53, %r52, %r51;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r54, %r53, 32767;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p6, %r51, %r51;
-; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r55, %r51, 4194304;
-; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r56, %r55, %r54, %p6;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r57, %rs9;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r58, %r57, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r59, %rs11;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r60, %r59, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r61, %r60, %r58;
-; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r62, %r61, 16, 1;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r63, %r62, %r61;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r64, %r63, 32767;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p7, %r61, %r61;
-; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r65, %r61, 4194304;
-; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r66, %r65, %r64, %p7;
-; SM60-NOFTZ-DISALLOW-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r67;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r68, %rs14;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r69, %r68, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r92;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r70, %rs16;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r71, %r70, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r72, %r71, %r69;
-; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r73, %r72, 16, 1;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r74, %r73, %r72;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r75, %r74, 32767;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p8, %r72, %r72;
-; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r76, %r72, 4194304;
-; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r77, %r76, %r75, %p8;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r78, %rs13;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r79, %r78, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r80, %rs15;
-; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r81, %r80, 16;
-; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r82, %r81, %r79;
-; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r83, %r82, 16, 1;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r84, %r83, %r82;
-; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r85, %r84, 32767;
-; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p9, %r82, %r82;
-; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r86, %r82, 4194304;
-; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r87, %r86, %r85, %p9;
-; SM60-NOFTZ-DISALLOW-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r51, %rs10;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r52, %r51, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r95;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r53, %rs12;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r54, %r53, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r55, %r54, %r52;
+; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r56, %r55, 16, 1;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r57, %r56, %r55;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r58, %r57, 32767;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p6, %r55, %r55;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r59, %r55, 4194304;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r60, %r59, %r58, %p6;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r61, %rs9;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r62, %r61, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r63, %rs11;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r64, %r63, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r65, %r64, %r62;
+; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r66, %r65, 16, 1;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r67, %r66, %r65;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r68, %r67, 32767;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p7, %r65, %r65;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r69, %r65, 4194304;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r70, %r69, %r68, %p7;
+; SM60-NOFTZ-DISALLOW-NEXT: prmt.b32 %r71, %r70, %r60, 0x7632U;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r71;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r72, %rs14;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r73, %r72, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r96;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r74, %rs16;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r75, %r74, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r76, %r75, %r73;
+; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r77, %r76, 16, 1;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r78, %r77, %r76;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r79, %r78, 32767;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p8, %r76, %r76;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r80, %r76, 4194304;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r81, %r80, %r79, %p8;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r82, %rs13;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r83, %r82, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r84, %rs15;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r85, %r84, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r86, %r85, %r83;
+; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r87, %r86, 16, 1;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r88, %r87, %r86;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r89, %r88, 32767;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p9, %r86, %r86;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r90, %r86, 4194304;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r91, %r90, %r89, %p9;
+; SM60-NOFTZ-DISALLOW-NEXT: prmt.b32 %r92, %r91, %r81, 0x7632U;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r92;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r91;
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r95;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r96;
; SM60-NOFTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NOFTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
-; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
-; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM60-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r96}, %rd21; }
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r95, %rd21;
; SM60-NOFTZ-DISALLOW-NEXT: @%p10 bra $L__BB319_3;
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r93, %r94, %r95, %r96};
; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
; SM60-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v8bf16_global_cta(
; SM60-NOFTZ-ALLOW: {
; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<11>;
; SM60-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<17>;
-; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<93>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<97>;
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<22>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: xor.b32 %r2, %r6, -2147450880;
+; SM60-NOFTZ-ALLOW-NEXT: xor.b32 %r1, %r5, -2147450880;
+; SM60-NOFTZ-ALLOW-NEXT: xor.b32 %r4, %r8, -2147450880;
+; SM60-NOFTZ-ALLOW-NEXT: xor.b32 %r3, %r7, -2147450880;
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r94}, %rd2; }
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r93, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NOFTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM60-NOFTZ-ALLOW-NEXT: $L__BB319_1: // %atomicrmw.start
; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r6, %r5, 16;
-; SM60-NOFTZ-ALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r89;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r7, %rs4;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r8, %r7, 16;
-; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r9, %r8, %r6;
-; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r10, %r9, 16, 1;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r11, %r10, %r9;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r12, %r11, 32767;
-; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r9, %r9;
-; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r13, %r9, 4194304;
-; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r14, %r13, %r12, %p1;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r15, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r16, %r15, 16;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r17, %rs3;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r18, %r17, 16;
-; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r19, %r18, %r16;
-; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r21, %r20, %r19;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r22, %r21, 32767;
-; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r23, %r19, 4194304;
-; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; SM60-NOFTZ-ALLOW-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r25;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r26, %rs6;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r27, %r26, 16;
-; SM60-NOFTZ-ALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r90;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r28, %rs8;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r29, %r28, 16;
-; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r30, %r29, %r27;
-; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r31, %r30, 16, 1;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r32, %r31, %r30;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r33, %r32, 32767;
-; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p3, %r30, %r30;
-; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r34, %r30, 4194304;
-; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r35, %r34, %r33, %p3;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r36, %rs5;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r37, %r36, 16;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r38, %rs7;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r39, %r38, 16;
-; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r40, %r39, %r37;
-; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r41, %r40, 16, 1;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r42, %r41, %r40;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r43, %r42, 32767;
-; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p4, %r40, %r40;
-; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r44, %r40, 4194304;
-; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r45, %r44, %r43, %p4;
-; SM60-NOFTZ-ALLOW-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs2;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r93;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r11, %rs4;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f32 %r13, %r12, %r10;
+; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r14, %r13, 16, 1;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, %r13;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r16, %r15, 32767;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r13, %r13;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r17, %r13, 4194304;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r19, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r20, %r19, 16;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r21, %rs3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r22, %r21, 16;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f32 %r23, %r22, %r20;
+; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r24, %r23, 16, 1;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r25, %r24, %r23;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r26, %r25, 32767;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r23, %r23;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r27, %r23, 4194304;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r28, %r27, %r26, %p2;
+; SM60-NOFTZ-ALLOW-NEXT: prmt.b32 %r29, %r28, %r18, 0x7632U;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r29;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r30, %rs6;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r31, %r30, 16;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r94;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r32, %rs8;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r33, %r32, 16;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f32 %r34, %r33, %r31;
+; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r35, %r34, 16, 1;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r36, %r35, %r34;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r37, %r36, 32767;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p3, %r34, %r34;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r38, %r34, 4194304;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r39, %r38, %r37, %p3;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r40, %rs5;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r41, %r40, 16;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r42, %rs7;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r43, %r42, 16;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f32 %r44, %r43, %r41;
+; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r45, %r44, 16, 1;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r46, %r45, %r44;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r47, %r46, 32767;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p4, %r44, %r44;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r48, %r44, 4194304;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r49, %r48, %r47, %p4;
+; SM60-NOFTZ-ALLOW-NEXT: prmt.b32 %r50, %r49, %r39, 0x7632U;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r50;
; SM60-NOFTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NOFTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r89;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r93;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r94;
; SM60-NOFTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NOFTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
-; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r94}, %rd11; }
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r93, %rd11;
; SM60-NOFTZ-ALLOW-NEXT: @%p5 bra $L__BB319_1;
; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r96}, %rd12; }
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r95, %rd12;
; SM60-NOFTZ-ALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM60-NOFTZ-ALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r4;
; SM60-NOFTZ-ALLOW-NEXT: $L__BB319_3: // %atomicrmw.start2
; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r47, %rs10;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r48, %r47, 16;
-; SM60-NOFTZ-ALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r91;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r49, %rs12;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r50, %r49, 16;
-; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r51, %r50, %r48;
-; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r52, %r51, 16, 1;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r53, %r52, %r51;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r54, %r53, 32767;
-; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p6, %r51, %r51;
-; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r55, %r51, 4194304;
-; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r56, %r55, %r54, %p6;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r57, %rs9;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r58, %r57, 16;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r59, %rs11;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r60, %r59, 16;
-; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r61, %r60, %r58;
-; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r62, %r61, 16, 1;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r63, %r62, %r61;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r64, %r63, 32767;
-; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p7, %r61, %r61;
-; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r65, %r61, 4194304;
-; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r66, %r65, %r64, %p7;
-; SM60-NOFTZ-ALLOW-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r67;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r68, %rs14;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r69, %r68, 16;
-; SM60-NOFTZ-ALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r92;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r70, %rs16;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r71, %r70, 16;
-; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r72, %r71, %r69;
-; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r73, %r72, 16, 1;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r74, %r73, %r72;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r75, %r74, 32767;
-; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p8, %r72, %r72;
-; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r76, %r72, 4194304;
-; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r77, %r76, %r75, %p8;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r78, %rs13;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r79, %r78, 16;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r80, %rs15;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r81, %r80, 16;
-; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r82, %r81, %r79;
-; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r83, %r82, 16, 1;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r84, %r83, %r82;
-; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r85, %r84, 32767;
-; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p9, %r82, %r82;
-; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r86, %r82, 4194304;
-; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r87, %r86, %r85, %p9;
-; SM60-NOFTZ-ALLOW-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r88;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM60-NOFTZ-ALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r91;
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r92;
-; SM60-NOFTZ-ALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
-; SM60-NOFTZ-ALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
-; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
-; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
-; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r91, %rd21;
-; SM60-NOFTZ-ALLOW-NEXT: @%p10 bra $L__BB319_3;
-; SM60-NOFTZ-ALLOW-NEXT: // %bb.4: // %atomicrmw.end1
-; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
-; SM60-NOFTZ-ALLOW-NEXT: ret;
-;
-; SM60-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8bf16_global_cta(
-; SM60-FTZ-DISALLOW: {
-; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<11>;
-; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<17>;
-; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<93>;
-; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<22>;
-; SM60-FTZ-DISALLOW-EMPTY:
-; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
-; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
-; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r89, %rd2;
-; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM60-FTZ-DISALLOW-NEXT: $L__BB319_1: // %atomicrmw.start
-; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r6, %r5, 16;
-; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r89;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r7, %rs4;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r8, %r7, 16;
-; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r9, %r8, %r6;
-; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r10, %r9, 16, 1;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r11, %r10, %r9;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r12, %r11, 32767;
-; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r9, %r9;
-; SM60-FTZ-DISALLOW-NEXT: or.b32 %r13, %r9, 4194304;
-; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r14, %r13, %r12, %p1;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r15, %rs1;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r16, %r15, 16;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r17, %rs3;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r18, %r17, 16;
-; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r19, %r18, %r16;
-; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r21, %r20, %r19;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r22, %r21, 32767;
-; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r19, %r19;
-; SM60-FTZ-DISALLOW-NEXT: or.b32 %r23, %r19, 4194304;
-; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; SM60-FTZ-DISALLOW-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r25;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r26, %rs6;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r27, %r26, 16;
-; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r90;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r28, %rs8;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r29, %r28, 16;
-; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r30, %r29, %r27;
-; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r31, %r30, 16, 1;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r32, %r31, %r30;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r33, %r32, 32767;
-; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p3, %r30, %r30;
-; SM60-FTZ-DISALLOW-NEXT: or.b32 %r34, %r30, 4194304;
-; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r35, %r34, %r33, %p3;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r36, %rs5;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r37, %r36, 16;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r38, %rs7;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r39, %r38, 16;
-; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r40, %r39, %r37;
-; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r41, %r40, 16, 1;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r42, %r41, %r40;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r43, %r42, 32767;
-; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p4, %r40, %r40;
-; SM60-FTZ-DISALLOW-NEXT: or.b32 %r44, %r40, 4194304;
-; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r45, %r44, %r43, %p4;
-; SM60-FTZ-DISALLOW-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r46;
-; SM60-FTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-FTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r89;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r90;
-; SM60-FTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-FTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-FTZ-DISALLOW-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
-; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r89, %rd11;
-; SM60-FTZ-DISALLOW-NEXT: @%p5 bra $L__BB319_1;
-; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r91, %rd12;
-; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r3;
-; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r4;
-; SM60-FTZ-DISALLOW-NEXT: $L__BB319_3: // %atomicrmw.start2
-; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r47, %rs10;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r48, %r47, 16;
-; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r91;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r49, %rs12;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r50, %r49, 16;
-; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r51, %r50, %r48;
-; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r52, %r51, 16, 1;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r53, %r52, %r51;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r54, %r53, 32767;
-; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p6, %r51, %r51;
-; SM60-FTZ-DISALLOW-NEXT: or.b32 %r55, %r51, 4194304;
-; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r56, %r55, %r54, %p6;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r57, %rs9;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r58, %r57, 16;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r59, %rs11;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r60, %r59, 16;
-; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r61, %r60, %r58;
-; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r62, %r61, 16, 1;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r63, %r62, %r61;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r64, %r63, 32767;
-; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p7, %r61, %r61;
-; SM60-FTZ-DISALLOW-NEXT: or.b32 %r65, %r61, 4194304;
-; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r66, %r65, %r64, %p7;
-; SM60-FTZ-DISALLOW-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r67;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r68, %rs14;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r69, %r68, 16;
-; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r92;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r70, %rs16;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r71, %r70, 16;
-; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r72, %r71, %r69;
-; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r73, %r72, 16, 1;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r74, %r73, %r72;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r75, %r74, 32767;
-; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p8, %r72, %r72;
-; SM60-FTZ-DISALLOW-NEXT: or.b32 %r76, %r72, 4194304;
-; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r77, %r76, %r75, %p8;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r78, %rs13;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r79, %r78, 16;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r80, %rs15;
-; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r81, %r80, 16;
-; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r82, %r81, %r79;
-; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r83, %r82, 16, 1;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r84, %r83, %r82;
-; SM60-FTZ-DISALLOW-NEXT: add.s32 %r85, %r84, 32767;
-; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p9, %r82, %r82;
-; SM60-FTZ-DISALLOW-NEXT: or.b32 %r86, %r82, 4194304;
-; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r87, %r86, %r85, %p9;
-; SM60-FTZ-DISALLOW-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r51, %rs10;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r52, %r51, 16;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r95;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r53, %rs12;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r54, %r53, 16;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f32 %r55, %r54, %r52;
+; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r56, %r55, 16, 1;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r57, %r56, %r55;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r58, %r57, 32767;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p6, %r55, %r55;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r59, %r55, 4194304;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r60, %r59, %r58, %p6;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r61, %rs9;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r62, %r61, 16;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r63, %rs11;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r64, %r63, 16;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f32 %r65, %r64, %r62;
+; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r66, %r65, 16, 1;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r67, %r66, %r65;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r68, %r67, 32767;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p7, %r65, %r65;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r69, %r65, 4194304;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r70, %r69, %r68, %p7;
+; SM60-NOFTZ-ALLOW-NEXT: prmt.b32 %r71, %r70, %r60, 0x7632U;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r71;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r72, %rs14;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r73, %r72, 16;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r96;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r74, %rs16;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r75, %r74, 16;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f32 %r76, %r75, %r73;
+; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r77, %r76, 16, 1;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r78, %r77, %r76;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r79, %r78, 32767;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p8, %r76, %r76;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r80, %r76, 4194304;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r81, %r80, %r79, %p8;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r82, %rs13;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r83, %r82, 16;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r84, %rs15;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r85, %r84, 16;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f32 %r86, %r85, %r83;
+; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r87, %r86, 16, 1;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r88, %r87, %r86;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r89, %r88, 32767;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p9, %r86, %r86;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r90, %r86, 4194304;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r91, %r90, %r89, %p9;
+; SM60-NOFTZ-ALLOW-NEXT: prmt.b32 %r92, %r91, %r81, 0x7632U;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r92;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NOFTZ-ALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r95;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r96;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NOFTZ-ALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM60-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r96}, %rd21; }
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r95, %rd21;
+; SM60-NOFTZ-ALLOW-NEXT: @%p10 bra $L__BB319_3;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r93, %r94, %r95, %r96};
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8bf16_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<11>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<17>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<97>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<22>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: xor.b32 %r2, %r6, -2147450880;
+; SM60-FTZ-DISALLOW-NEXT: xor.b32 %r1, %r5, -2147450880;
+; SM60-FTZ-DISALLOW-NEXT: xor.b32 %r4, %r8, -2147450880;
+; SM60-FTZ-DISALLOW-NEXT: xor.b32 %r3, %r7, -2147450880;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r94}, %rd2; }
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r93, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-FTZ-DISALLOW-NEXT: $L__BB319_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs2;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r93;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r11, %rs4;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r13, %r12, %r10;
+; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r14, %r13, 16, 1;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, %r13;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r16, %r15, 32767;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r13, %r13;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r17, %r13, 4194304;
+; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r19, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r20, %r19, 16;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r21, %rs3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r22, %r21, 16;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r23, %r22, %r20;
+; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r24, %r23, 16, 1;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r25, %r24, %r23;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r26, %r25, 32767;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r23, %r23;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r27, %r23, 4194304;
+; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r28, %r27, %r26, %p2;
+; SM60-FTZ-DISALLOW-NEXT: prmt.b32 %r29, %r28, %r18, 0x7632U;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r29;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r30, %rs6;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r31, %r30, 16;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r94;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r32, %rs8;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r33, %r32, 16;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r34, %r33, %r31;
+; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r35, %r34, 16, 1;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r36, %r35, %r34;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r37, %r36, 32767;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p3, %r34, %r34;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r38, %r34, 4194304;
+; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r39, %r38, %r37, %p3;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r40, %rs5;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r41, %r40, 16;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r42, %rs7;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r43, %r42, 16;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r44, %r43, %r41;
+; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r45, %r44, 16, 1;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r46, %r45, %r44;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r47, %r46, 32767;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p4, %r44, %r44;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r48, %r44, 4194304;
+; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r49, %r48, %r47, %p4;
+; SM60-FTZ-DISALLOW-NEXT: prmt.b32 %r50, %r49, %r39, 0x7632U;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM60-FTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-FTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r93;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r94;
+; SM60-FTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-FTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r94}, %rd11; }
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r93, %rd11;
+; SM60-FTZ-DISALLOW-NEXT: @%p5 bra $L__BB319_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r96}, %rd12; }
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r95, %rd12;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM60-FTZ-DISALLOW-NEXT: $L__BB319_3: // %atomicrmw.start2
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r51, %rs10;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r52, %r51, 16;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r95;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r53, %rs12;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r54, %r53, 16;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r55, %r54, %r52;
+; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r56, %r55, 16, 1;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r57, %r56, %r55;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r58, %r57, 32767;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p6, %r55, %r55;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r59, %r55, 4194304;
+; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r60, %r59, %r58, %p6;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r61, %rs9;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r62, %r61, 16;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r63, %rs11;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r64, %r63, 16;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r65, %r64, %r62;
+; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r66, %r65, 16, 1;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r67, %r66, %r65;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r68, %r67, 32767;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p7, %r65, %r65;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r69, %r65, 4194304;
+; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r70, %r69, %r68, %p7;
+; SM60-FTZ-DISALLOW-NEXT: prmt.b32 %r71, %r70, %r60, 0x7632U;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r71;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r72, %rs14;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r73, %r72, 16;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r96;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r74, %rs16;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r75, %r74, 16;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r76, %r75, %r73;
+; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r77, %r76, 16, 1;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r78, %r77, %r76;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r79, %r78, 32767;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p8, %r76, %r76;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r80, %r76, 4194304;
+; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r81, %r80, %r79, %p8;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r82, %rs13;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r83, %r82, 16;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r84, %rs15;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r85, %r84, 16;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r86, %r85, %r83;
+; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r87, %r86, 16, 1;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r88, %r87, %r86;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r89, %r88, 32767;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p9, %r86, %r86;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r90, %r86, 4194304;
+; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r91, %r90, %r89, %p9;
+; SM60-FTZ-DISALLOW-NEXT: prmt.b32 %r92, %r91, %r81, 0x7632U;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r92;
; SM60-FTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-FTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r91;
-; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r95;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r96;
; SM60-FTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-FTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM60-FTZ-DISALLOW-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
-; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
-; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM60-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r96}, %rd21; }
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r95, %rd21;
; SM60-FTZ-DISALLOW-NEXT: @%p10 bra $L__BB319_3;
; SM60-FTZ-DISALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM60-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r93, %r94, %r95, %r96};
; SM60-FTZ-DISALLOW-NEXT: ret;
;
; SM60-FTZ-ALLOW-LABEL: fsub_acq_rel_v8bf16_global_cta(
; SM60-FTZ-ALLOW: {
; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<11>;
; SM60-FTZ-ALLOW-NEXT: .reg .b16 %rs<17>;
-; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<93>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<97>;
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<22>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: xor.b32 %r2, %r6, -2147450880;
+; SM60-FTZ-ALLOW-NEXT: xor.b32 %r1, %r5, -2147450880;
+; SM60-FTZ-ALLOW-NEXT: xor.b32 %r4, %r8, -2147450880;
+; SM60-FTZ-ALLOW-NEXT: xor.b32 %r3, %r7, -2147450880;
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r94}, %rd2; }
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r93, %rd2;
; SM60-FTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-FTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM60-FTZ-ALLOW-NEXT: $L__BB319_1: // %atomicrmw.start
; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r6, %r5, 16;
-; SM60-FTZ-ALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r89;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r7, %rs4;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r8, %r7, 16;
-; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r9, %r8, %r6;
-; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r10, %r9, 16, 1;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r11, %r10, %r9;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r12, %r11, 32767;
-; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r9, %r9;
-; SM60-FTZ-ALLOW-NEXT: or.b32 %r13, %r9, 4194304;
-; SM60-FTZ-ALLOW-NEXT: selp.b32 %r14, %r13, %r12, %p1;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r15, %rs1;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r16, %r15, 16;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r17, %rs3;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r18, %r17, 16;
-; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r19, %r18, %r16;
-; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r21, %r20, %r19;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r22, %r21, 32767;
-; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r19, %r19;
-; SM60-FTZ-ALLOW-NEXT: or.b32 %r23, %r19, 4194304;
-; SM60-FTZ-ALLOW-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; SM60-FTZ-ALLOW-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r25;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r26, %rs6;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r27, %r26, 16;
-; SM60-FTZ-ALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r90;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r28, %rs8;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r29, %r28, 16;
-; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r30, %r29, %r27;
-; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r31, %r30, 16, 1;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r32, %r31, %r30;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r33, %r32, 32767;
-; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p3, %r30, %r30;
-; SM60-FTZ-ALLOW-NEXT: or.b32 %r34, %r30, 4194304;
-; SM60-FTZ-ALLOW-NEXT: selp.b32 %r35, %r34, %r33, %p3;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r36, %rs5;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r37, %r36, 16;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r38, %rs7;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r39, %r38, 16;
-; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r40, %r39, %r37;
-; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r41, %r40, 16, 1;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r42, %r41, %r40;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r43, %r42, 32767;
-; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p4, %r40, %r40;
-; SM60-FTZ-ALLOW-NEXT: or.b32 %r44, %r40, 4194304;
-; SM60-FTZ-ALLOW-NEXT: selp.b32 %r45, %r44, %r43, %p4;
-; SM60-FTZ-ALLOW-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs2;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r93;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r11, %rs4;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r13, %r12, %r10;
+; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r14, %r13, 16, 1;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, %r13;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r16, %r15, 32767;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r13, %r13;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r17, %r13, 4194304;
+; SM60-FTZ-ALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r19, %rs1;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r20, %r19, 16;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r21, %rs3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r22, %r21, 16;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r23, %r22, %r20;
+; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r24, %r23, 16, 1;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r25, %r24, %r23;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r26, %r25, 32767;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r23, %r23;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r27, %r23, 4194304;
+; SM60-FTZ-ALLOW-NEXT: selp.b32 %r28, %r27, %r26, %p2;
+; SM60-FTZ-ALLOW-NEXT: prmt.b32 %r29, %r28, %r18, 0x7632U;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r29;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r30, %rs6;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r31, %r30, 16;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r94;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r32, %rs8;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r33, %r32, 16;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r34, %r33, %r31;
+; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r35, %r34, 16, 1;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r36, %r35, %r34;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r37, %r36, 32767;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p3, %r34, %r34;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r38, %r34, 4194304;
+; SM60-FTZ-ALLOW-NEXT: selp.b32 %r39, %r38, %r37, %p3;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r40, %rs5;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r41, %r40, 16;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r42, %rs7;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r43, %r42, 16;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r44, %r43, %r41;
+; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r45, %r44, 16, 1;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r46, %r45, %r44;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r47, %r46, 32767;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p4, %r44, %r44;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r48, %r44, 4194304;
+; SM60-FTZ-ALLOW-NEXT: selp.b32 %r49, %r48, %r47, %p4;
+; SM60-FTZ-ALLOW-NEXT: prmt.b32 %r50, %r49, %r39, 0x7632U;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r50;
; SM60-FTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-FTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r89;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r93;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r94;
; SM60-FTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-FTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM60-FTZ-ALLOW-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
-; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r94}, %rd11; }
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r93, %rd11;
; SM60-FTZ-ALLOW-NEXT: @%p5 bra $L__BB319_1;
; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r96}, %rd12; }
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r95, %rd12;
; SM60-FTZ-ALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM60-FTZ-ALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r4;
; SM60-FTZ-ALLOW-NEXT: $L__BB319_3: // %atomicrmw.start2
; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r47, %rs10;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r48, %r47, 16;
-; SM60-FTZ-ALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r91;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r49, %rs12;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r50, %r49, 16;
-; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r51, %r50, %r48;
-; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r52, %r51, 16, 1;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r53, %r52, %r51;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r54, %r53, 32767;
-; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p6, %r51, %r51;
-; SM60-FTZ-ALLOW-NEXT: or.b32 %r55, %r51, 4194304;
-; SM60-FTZ-ALLOW-NEXT: selp.b32 %r56, %r55, %r54, %p6;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r57, %rs9;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r58, %r57, 16;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r59, %rs11;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r60, %r59, 16;
-; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r61, %r60, %r58;
-; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r62, %r61, 16, 1;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r63, %r62, %r61;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r64, %r63, 32767;
-; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p7, %r61, %r61;
-; SM60-FTZ-ALLOW-NEXT: or.b32 %r65, %r61, 4194304;
-; SM60-FTZ-ALLOW-NEXT: selp.b32 %r66, %r65, %r64, %p7;
-; SM60-FTZ-ALLOW-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r67;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r68, %rs14;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r69, %r68, 16;
-; SM60-FTZ-ALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r92;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r70, %rs16;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r71, %r70, 16;
-; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r72, %r71, %r69;
-; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r73, %r72, 16, 1;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r74, %r73, %r72;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r75, %r74, 32767;
-; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p8, %r72, %r72;
-; SM60-FTZ-ALLOW-NEXT: or.b32 %r76, %r72, 4194304;
-; SM60-FTZ-ALLOW-NEXT: selp.b32 %r77, %r76, %r75, %p8;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r78, %rs13;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r79, %r78, 16;
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r80, %rs15;
-; SM60-FTZ-ALLOW-NEXT: shl.b32 %r81, %r80, 16;
-; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r82, %r81, %r79;
-; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r83, %r82, 16, 1;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r84, %r83, %r82;
-; SM60-FTZ-ALLOW-NEXT: add.s32 %r85, %r84, 32767;
-; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p9, %r82, %r82;
-; SM60-FTZ-ALLOW-NEXT: or.b32 %r86, %r82, 4194304;
-; SM60-FTZ-ALLOW-NEXT: selp.b32 %r87, %r86, %r85, %p9;
-; SM60-FTZ-ALLOW-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r51, %rs10;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r52, %r51, 16;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r95;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r53, %rs12;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r54, %r53, 16;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r55, %r54, %r52;
+; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r56, %r55, 16, 1;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r57, %r56, %r55;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r58, %r57, 32767;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p6, %r55, %r55;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r59, %r55, 4194304;
+; SM60-FTZ-ALLOW-NEXT: selp.b32 %r60, %r59, %r58, %p6;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r61, %rs9;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r62, %r61, 16;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r63, %rs11;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r64, %r63, 16;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r65, %r64, %r62;
+; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r66, %r65, 16, 1;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r67, %r66, %r65;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r68, %r67, 32767;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p7, %r65, %r65;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r69, %r65, 4194304;
+; SM60-FTZ-ALLOW-NEXT: selp.b32 %r70, %r69, %r68, %p7;
+; SM60-FTZ-ALLOW-NEXT: prmt.b32 %r71, %r70, %r60, 0x7632U;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r71;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r72, %rs14;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r73, %r72, 16;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r96;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r74, %rs16;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r75, %r74, 16;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r76, %r75, %r73;
+; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r77, %r76, 16, 1;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r78, %r77, %r76;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r79, %r78, 32767;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p8, %r76, %r76;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r80, %r76, 4194304;
+; SM60-FTZ-ALLOW-NEXT: selp.b32 %r81, %r80, %r79, %p8;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r82, %rs13;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r83, %r82, 16;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r84, %rs15;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r85, %r84, 16;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r86, %r85, %r83;
+; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r87, %r86, 16, 1;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r88, %r87, %r86;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r89, %r88, 32767;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p9, %r86, %r86;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r90, %r86, 4194304;
+; SM60-FTZ-ALLOW-NEXT: selp.b32 %r91, %r90, %r89, %p9;
+; SM60-FTZ-ALLOW-NEXT: prmt.b32 %r92, %r91, %r81, 0x7632U;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r92;
; SM60-FTZ-ALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-FTZ-ALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r91;
-; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r95;
+; SM60-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r96;
; SM60-FTZ-ALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-FTZ-ALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM60-FTZ-ALLOW-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
-; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
-; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM60-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r96}, %rd21; }
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r95, %rd21;
; SM60-FTZ-ALLOW-NEXT: @%p10 bra $L__BB319_3;
; SM60-FTZ-ALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM60-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r93, %r94, %r95, %r96};
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -24342,9 +24404,9 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -24387,9 +24449,9 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -24432,9 +24494,9 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -24477,9 +24539,9 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -25678,9 +25740,9 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -25723,9 +25785,9 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -25768,9 +25830,9 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -25813,9 +25875,9 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -27014,9 +27076,9 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -27061,9 +27123,9 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -27108,9 +27170,9 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -27155,9 +27217,9 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -28414,9 +28476,9 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -28461,9 +28523,9 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
-; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -28508,9 +28570,9 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM60-FTZ-DISALLOW-NEXT: membar.cta;
-; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -28555,9 +28617,9 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM60-FTZ-ALLOW-NEXT: membar.cta;
-; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -29814,14 +29876,14 @@ define <1 x i8> @add_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd2, [add_monotonic_v1i8_global_cta_param_0];
-; SM60-NEXT: ld.param.b8 %r6, [add_monotonic_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: and.b32 %r8, %r7, 3;
-; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
; SM60-NEXT: shl.b32 %r2, 255, %r1;
; SM60-NEXT: not.b32 %r3, %r2;
-; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.param.b8 %r8, [add_monotonic_v1i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r4, %r8, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
; SM60-NEXT: $L__BB336_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
@@ -30038,14 +30100,14 @@ define <1 x i8> @add_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd2, [add_acquire_v1i8_global_cta_param_0];
-; SM60-NEXT: ld.param.b8 %r6, [add_acquire_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: and.b32 %r8, %r7, 3;
-; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
; SM60-NEXT: shl.b32 %r2, 255, %r1;
; SM60-NEXT: not.b32 %r3, %r2;
-; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.param.b8 %r8, [add_acquire_v1i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r4, %r8, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
; SM60-NEXT: $L__BB340_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
@@ -30267,14 +30329,14 @@ define <1 x i8> @add_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd2, [add_release_v1i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b8 %r6, [add_release_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: and.b32 %r8, %r7, 3;
-; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
; SM60-NEXT: shl.b32 %r2, 255, %r1;
; SM60-NEXT: not.b32 %r3, %r2;
-; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.param.b8 %r8, [add_release_v1i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r4, %r8, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
; SM60-NEXT: $L__BB344_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
@@ -30733,14 +30795,14 @@ define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd2, [nand_monotonic_v1i8_global_cta_param_0];
-; SM60-NEXT: ld.param.b8 %r6, [nand_monotonic_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: and.b32 %r8, %r7, 3;
-; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
; SM60-NEXT: shl.b32 %r2, 255, %r1;
; SM60-NEXT: not.b32 %r3, %r2;
-; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.param.b8 %r8, [nand_monotonic_v1i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r4, %r8, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
; SM60-NEXT: $L__BB360_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
@@ -30872,14 +30934,14 @@ define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd2, [nand_acquire_v1i8_global_cta_param_0];
-; SM60-NEXT: ld.param.b8 %r6, [nand_acquire_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: and.b32 %r8, %r7, 3;
-; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
; SM60-NEXT: shl.b32 %r2, 255, %r1;
; SM60-NEXT: not.b32 %r3, %r2;
-; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.param.b8 %r8, [nand_acquire_v1i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r4, %r8, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
; SM60-NEXT: $L__BB364_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
@@ -31016,14 +31078,14 @@ define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd2, [nand_release_v1i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b8 %r6, [nand_release_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: and.b32 %r8, %r7, 3;
-; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
; SM60-NEXT: shl.b32 %r2, 255, %r1;
; SM60-NEXT: not.b32 %r3, %r2;
-; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.param.b8 %r8, [nand_release_v1i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r4, %r8, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
; SM60-NEXT: $L__BB368_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
@@ -31157,19 +31219,19 @@ define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_monotonic_v1i32_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
-; SM60-NEXT: ld.param.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
; SM60-NEXT: $L__BB372_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
; SM60-NEXT: not.b32 %r4, %r3;
-; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
; SM60-NEXT: @%p1 bra $L__BB372_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
ret <1 x i32> %retval
@@ -31387,20 +31449,20 @@ define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_acquire_v1i32_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
-; SM60-NEXT: ld.param.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
; SM60-NEXT: $L__BB376_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
; SM60-NEXT: not.b32 %r4, %r3;
-; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
; SM60-NEXT: @%p1 bra $L__BB376_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
ret <1 x i32> %retval
@@ -31621,20 +31683,20 @@ define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_release_v1i32_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v1i32_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b32 %r1, [nand_release_v1i32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
; SM60-NEXT: $L__BB380_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
; SM60-NEXT: not.b32 %r4, %r3;
-; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
; SM60-NEXT: @%p1 bra $L__BB380_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
ret <1 x i32> %retval
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
index c5cebdf5946657..d7a72b76ecf7d7 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
@@ -22,14 +22,14 @@ define <1 x i8> @xchg_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b8 %r5, [xchg_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
-; SM70-NEXT: and.b32 %r7, %r6, 3;
-; SM70-NEXT: shl.b32 %r1, %r7, 3;
-; SM70-NEXT: shl.b32 %r8, 255, %r1;
-; SM70-NEXT: not.b32 %r2, %r8;
-; SM70-NEXT: shl.b32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: shl.b32 %r7, 255, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.param.b8 %r8, [xchg_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r3, %r8, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-NEXT: $L__BB0_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
@@ -124,8 +124,8 @@ define <1 x i16> @xchg_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i16_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
; SM70-NEXT: and.b32 %r7, %r6, 3;
@@ -217,10 +217,8 @@ define <1 x i32> @xchg_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r2, [%rd1], %r1;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -315,10 +313,8 @@ define <1 x i64> @xchg_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd3, [%rd1], %rd2;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -405,14 +401,14 @@ define <1 x i8> @add_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b8 %r6, [add_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: and.b32 %r8, %r7, 3;
-; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
; SM70-NEXT: shl.b32 %r2, 255, %r1;
; SM70-NEXT: not.b32 %r3, %r2;
-; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.param.b8 %r8, [add_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r4, %r8, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT: $L__BB16_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
@@ -830,10 +826,8 @@ define <1 x i32> @add_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [add_acq_rel_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.add.u32 %r2, [%rd1], %r1;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -914,10 +908,8 @@ define <1 x i64> @add_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd3, [%rd1], %rd2;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.add.u64 %rd3, [%rd1], %rd2;
; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -1004,14 +996,14 @@ define <1 x i8> @sub_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b8 %r6, [sub_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: and.b32 %r8, %r7, 3;
-; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
; SM70-NEXT: shl.b32 %r2, 255, %r1;
; SM70-NEXT: not.b32 %r3, %r2;
-; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.param.b8 %r8, [sub_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r4, %r8, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT: $L__BB32_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
@@ -1348,74 +1340,90 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-LABEL: sub_acq_rel_v8i16_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<3>;
-; SM70-NEXT: .reg .b16 %rs<25>;
-; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b16 %rs<41>;
+; SM70-NEXT: .reg .b32 %r<17>;
; SM70-NEXT: .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r6;
+; SM70-NEXT: neg.s16 %rs3, %rs2;
+; SM70-NEXT: neg.s16 %rs4, %rs1;
+; SM70-NEXT: mov.b32 %r2, {%rs4, %rs3};
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r5;
+; SM70-NEXT: neg.s16 %rs7, %rs6;
+; SM70-NEXT: neg.s16 %rs8, %rs5;
+; SM70-NEXT: mov.b32 %r1, {%rs8, %rs7};
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r8;
+; SM70-NEXT: neg.s16 %rs11, %rs10;
+; SM70-NEXT: neg.s16 %rs12, %rs9;
+; SM70-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r7;
+; SM70-NEXT: neg.s16 %rs15, %rs14;
+; SM70-NEXT: neg.s16 %rs16, %rs13;
+; SM70-NEXT: mov.b32 %r3, {%rs16, %rs15};
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r1;
+; SM70-NEXT: mov.b32 {%rs23, %rs24}, %r2;
; SM70-NEXT: $L__BB39_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
-; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
-; SM70-NEXT: sub.s16 %rs6, %rs3, %rs1;
-; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
-; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r10;
-; SM70-NEXT: sub.s16 %rs11, %rs10, %rs8;
-; SM70-NEXT: sub.s16 %rs12, %rs9, %rs7;
-; SM70-NEXT: mov.b32 %r6, {%rs12, %rs11};
-; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r13;
+; SM70-NEXT: add.s16 %rs21, %rs20, %rs18;
+; SM70-NEXT: add.s16 %rs22, %rs19, %rs17;
+; SM70-NEXT: mov.b32 %r9, {%rs22, %rs21};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: mov.b32 {%rs25, %rs26}, %r14;
+; SM70-NEXT: add.s16 %rs27, %rs26, %rs24;
+; SM70-NEXT: add.s16 %rs28, %rs25, %rs23;
+; SM70-NEXT: mov.b32 %r10, {%rs28, %rs27};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r14;
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
; SM70-NEXT: @%p1 bra $L__BB39_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
-; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM70-NEXT: mov.b32 {%rs29, %rs30}, %r3;
+; SM70-NEXT: mov.b32 {%rs35, %rs36}, %r4;
; SM70-NEXT: $L__BB39_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
-; SM70-NEXT: sub.s16 %rs17, %rs16, %rs14;
-; SM70-NEXT: sub.s16 %rs18, %rs15, %rs13;
-; SM70-NEXT: mov.b32 %r7, {%rs18, %rs17};
-; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
-; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
-; SM70-NEXT: sub.s16 %rs23, %rs22, %rs20;
-; SM70-NEXT: sub.s16 %rs24, %rs21, %rs19;
-; SM70-NEXT: mov.b32 %r8, {%rs24, %rs23};
-; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: mov.b32 {%rs31, %rs32}, %r15;
+; SM70-NEXT: add.s16 %rs33, %rs32, %rs30;
+; SM70-NEXT: add.s16 %rs34, %rs31, %rs29;
+; SM70-NEXT: mov.b32 %r11, {%rs34, %rs33};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: mov.b32 {%rs37, %rs38}, %r16;
+; SM70-NEXT: add.s16 %rs39, %rs38, %rs36;
+; SM70-NEXT: add.s16 %rs40, %rs37, %rs35;
+; SM70-NEXT: mov.b32 %r12, {%rs40, %rs39};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r16;
; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
-; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd21;
; SM70-NEXT: @%p2 bra $L__BB39_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -1429,11 +1437,9 @@ define <1 x i32> @sub_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [sub_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT: neg.s32 %r2, %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r2;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.add.u32 %r3, [%rd1], %r2;
; SM70-NEXT: st.param.b32 [func_retval0], %r3;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -1448,14 +1454,14 @@ define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v2i32_global_cta_param_1];
-; SM70-NEXT: neg.s32 %r3, %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1], %r3;
-; SM70-NEXT: neg.s32 %r5, %r2;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r5;
+; SM70-NEXT: neg.s32 %r3, %r2;
+; SM70-NEXT: neg.s32 %r4, %r1;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r4;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -1469,18 +1475,18 @@ define <4 x i32> @sub_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v4i32_global_cta_param_1];
-; SM70-NEXT: neg.s32 %r5, %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1], %r5;
+; SM70-NEXT: neg.s32 %r5, %r4;
+; SM70-NEXT: neg.s32 %r6, %r3;
; SM70-NEXT: neg.s32 %r7, %r2;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+4], %r7;
-; SM70-NEXT: neg.s32 %r9, %r3;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+8], %r9;
-; SM70-NEXT: neg.s32 %r11, %r4;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r11;
+; SM70-NEXT: neg.s32 %r8, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r5;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -1494,28 +1500,28 @@ define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i32_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i32_global_cta_param_1];
-; SM70-NEXT: neg.s32 %r9, %r5;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1], %r9;
-; SM70-NEXT: neg.s32 %r11, %r6;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+4], %r11;
-; SM70-NEXT: neg.s32 %r13, %r7;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+8], %r13;
-; SM70-NEXT: neg.s32 %r15, %r8;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+12], %r15;
-; SM70-NEXT: neg.s32 %r17, %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r18, [%rd1+16], %r17;
-; SM70-NEXT: neg.s32 %r19, %r2;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r20, [%rd1+20], %r19;
-; SM70-NEXT: neg.s32 %r21, %r3;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r22, [%rd1+24], %r21;
-; SM70-NEXT: neg.s32 %r23, %r4;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r24, [%rd1+28], %r23;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM70-NEXT: neg.s32 %r5, %r4;
+; SM70-NEXT: neg.s32 %r6, %r3;
+; SM70-NEXT: neg.s32 %r7, %r2;
+; SM70-NEXT: neg.s32 %r8, %r1;
+; SM70-NEXT: ld.param.v4.b32 {%r9, %r10, %r11, %r12}, [sub_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: neg.s32 %r13, %r12;
+; SM70-NEXT: neg.s32 %r14, %r11;
+; SM70-NEXT: neg.s32 %r15, %r10;
+; SM70-NEXT: neg.s32 %r16, %r9;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r17, [%rd1], %r16;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r18, [%rd1+4], %r15;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r19, [%rd1+8], %r14;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r20, [%rd1+12], %r13;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r21, [%rd1+16], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r22, [%rd1+20], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r23, [%rd1+24], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r24, [%rd1+28], %r5;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -1528,11 +1534,9 @@ define <1 x i64> @sub_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT: neg.s64 %rd3, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd3;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.add.u64 %rd4, [%rd1], %rd3;
; SM70-NEXT: st.param.b64 [func_retval0], %rd4;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -1546,14 +1550,14 @@ define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v2i64_global_cta_param_1];
-; SM70-NEXT: neg.s64 %rd4, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd5, [%rd1], %rd4;
-; SM70-NEXT: neg.s64 %rd6, %rd3;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd6;
+; SM70-NEXT: neg.s64 %rd4, %rd3;
+; SM70-NEXT: neg.s64 %rd5, %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd6, [%rd1], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd4;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd7};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -1566,20 +1570,20 @@ define <4 x i64> @sub_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v4i64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v4i64_global_cta_param_1];
-; SM70-NEXT: neg.s64 %rd6, %rd4;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1], %rd6;
-; SM70-NEXT: neg.s64 %rd8, %rd5;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd9, [%rd1+8], %rd8;
-; SM70-NEXT: neg.s64 %rd10, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1+16], %rd10;
-; SM70-NEXT: neg.s64 %rd12, %rd3;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd12;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd11, %rd13};
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd7, %rd9};
+; SM70-NEXT: neg.s64 %rd4, %rd3;
+; SM70-NEXT: neg.s64 %rd5, %rd2;
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: neg.s64 %rd8, %rd7;
+; SM70-NEXT: neg.s64 %rd9, %rd6;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd10, [%rd1], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1+8], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd12, [%rd1+16], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -1592,32 +1596,32 @@ define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v8i64_global_cta_param_1+48];
-; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v8i64_global_cta_param_1+32];
-; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [sub_acq_rel_v8i64_global_cta_param_1];
-; SM70-NEXT: neg.s64 %rd10, %rd8;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1], %rd10;
-; SM70-NEXT: neg.s64 %rd12, %rd9;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+8], %rd12;
-; SM70-NEXT: neg.s64 %rd14, %rd6;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd15, [%rd1+16], %rd14;
-; SM70-NEXT: neg.s64 %rd16, %rd7;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd17, [%rd1+24], %rd16;
-; SM70-NEXT: neg.s64 %rd18, %rd4;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd19, [%rd1+32], %rd18;
-; SM70-NEXT: neg.s64 %rd20, %rd5;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd21, [%rd1+40], %rd20;
-; SM70-NEXT: neg.s64 %rd22, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd23, [%rd1+48], %rd22;
-; SM70-NEXT: neg.s64 %rd24, %rd3;
-; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd25, [%rd1+56], %rd24;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd23, %rd25};
-; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd19, %rd21};
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd15, %rd17};
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd11, %rd13};
+; SM70-NEXT: neg.s64 %rd4, %rd3;
+; SM70-NEXT: neg.s64 %rd5, %rd2;
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: neg.s64 %rd8, %rd7;
+; SM70-NEXT: neg.s64 %rd9, %rd6;
+; SM70-NEXT: ld.param.v2.b64 {%rd10, %rd11}, [sub_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: neg.s64 %rd12, %rd11;
+; SM70-NEXT: neg.s64 %rd13, %rd10;
+; SM70-NEXT: ld.param.v2.b64 {%rd14, %rd15}, [sub_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: neg.s64 %rd16, %rd15;
+; SM70-NEXT: neg.s64 %rd17, %rd14;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd18, [%rd1], %rd17;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd19, [%rd1+8], %rd16;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd20, [%rd1+16], %rd13;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd21, [%rd1+24], %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd22, [%rd1+32], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd23, [%rd1+40], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd24, [%rd1+48], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd25, [%rd1+56], %rd4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd24, %rd25};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd22, %rd23};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd20, %rd21};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -1632,17 +1636,17 @@ define <1 x i8> @and_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b8 %r1, [and_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd2, %rd1, -4;
-; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
-; SM70-NEXT: and.b32 %r3, %r2, 3;
-; SM70-NEXT: shl.b32 %r4, %r3, 3;
-; SM70-NEXT: shl.b32 %r5, 255, %r4;
-; SM70-NEXT: not.b32 %r6, %r5;
-; SM70-NEXT: shl.b32 %r7, %r1, %r4;
-; SM70-NEXT: or.b32 %r8, %r7, %r6;
+; SM70-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM70-NEXT: and.b32 %r2, %r1, 3;
+; SM70-NEXT: shl.b32 %r3, %r2, 3;
+; SM70-NEXT: shl.b32 %r4, 255, %r3;
+; SM70-NEXT: not.b32 %r5, %r4;
+; SM70-NEXT: ld.param.b8 %r6, [and_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r7, %r6, %r3;
+; SM70-NEXT: or.b32 %r8, %r7, %r5;
; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r9, [%rd2], %r8;
-; SM70-NEXT: shr.u32 %r10, %r9, %r4;
+; SM70-NEXT: shr.u32 %r10, %r9, %r3;
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b8 [func_retval0], %r10;
; SM70-NEXT: ret;
@@ -1904,10 +1908,8 @@ define <1 x i32> @and_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [and_acq_rel_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r2, [%rd1], %r1;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.and.b32 %r2, [%rd1], %r1;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -1988,10 +1990,8 @@ define <1 x i64> @and_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd2, [and_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd3, [%rd1], %rd2;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.and.b64 %rd3, [%rd1], %rd2;
; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -2078,14 +2078,14 @@ define <1 x i8> @nand_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b8 %r6, [nand_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: and.b32 %r8, %r7, 3;
-; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
; SM70-NEXT: shl.b32 %r2, 255, %r1;
; SM70-NEXT: not.b32 %r3, %r2;
-; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.param.b8 %r8, [nand_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r4, %r8, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT: $L__BB64_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
@@ -2395,21 +2395,21 @@ define <1 x i32> @nand_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b32 %r1, [nand_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT: $L__BB72_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
; SM70-NEXT: not.b32 %r4, %r3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
; SM70-NEXT: @%p1 bra $L__BB72_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -2632,21 +2632,21 @@ define <1 x i64> @nand_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM70-NEXT: .reg .b64 %rd<7>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd2];
; SM70-NEXT: $L__BB76_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b64 %rd4, %rd6, %rd1;
+; SM70-NEXT: and.b64 %rd4, %rd6, %rd3;
; SM70-NEXT: not.b64 %rd5, %rd4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
-; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
-; SM70-NEXT: mov.b64 %rd6, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5;
+; SM70-NEXT: setp.ne.b64 %p1, %rd1, %rd6;
+; SM70-NEXT: mov.b64 %rd6, %rd1;
; SM70-NEXT: @%p1 bra $L__BB76_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -2661,29 +2661,29 @@ define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd5, [nand_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [nand_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
; SM70-NEXT: $L__BB77_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd3, %rd10;
-; SM70-NEXT: and.b64 %rd6, %rd3, %rd2;
+; SM70-NEXT: and.b64 %rd6, %rd10, %rd1;
; SM70-NEXT: not.b64 %rd7, %rd6;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd10, [%rd5], %rd3, %rd7;
-; SM70-NEXT: setp.ne.b64 %p1, %rd10, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
+; SM70-NEXT: mov.b64 %rd10, %rd3;
; SM70-NEXT: @%p1 bra $L__BB77_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
-; SM70-NEXT: $L__BB77_3: // %atomicrmw.start4
+; SM70-NEXT: $L__BB77_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b64 %rd8, %rd11, %rd1;
+; SM70-NEXT: and.b64 %rd8, %rd11, %rd2;
; SM70-NEXT: not.b64 %rd9, %rd8;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
; SM70-NEXT: mov.b64 %rd11, %rd4;
; SM70-NEXT: @%p2 bra $L__BB77_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end3
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd4};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -2698,50 +2698,50 @@ define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd9, [nand_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [nand_acq_rel_v4i64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
; SM70-NEXT: $L__BB78_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd4, %rd18;
-; SM70-NEXT: and.b64 %rd10, %rd4, %rd3;
+; SM70-NEXT: mov.b64 %rd5, %rd18;
+; SM70-NEXT: and.b64 %rd10, %rd5, %rd3;
; SM70-NEXT: not.b64 %rd11, %rd10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd4, %rd11;
-; SM70-NEXT: setp.ne.b64 %p1, %rd18, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
; SM70-NEXT: @%p1 bra $L__BB78_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
-; SM70-NEXT: $L__BB78_3: // %atomicrmw.start8
+; SM70-NEXT: $L__BB78_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd19;
-; SM70-NEXT: and.b64 %rd12, %rd5, %rd2;
+; SM70-NEXT: mov.b64 %rd6, %rd19;
+; SM70-NEXT: and.b64 %rd12, %rd6, %rd4;
; SM70-NEXT: not.b64 %rd13, %rd12;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd5, %rd13;
-; SM70-NEXT: setp.ne.b64 %p2, %rd19, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM70-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
; SM70-NEXT: @%p2 bra $L__BB78_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end7
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
-; SM70-NEXT: $L__BB78_5: // %atomicrmw.start20
+; SM70-NEXT: $L__BB78_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd7, %rd20;
-; SM70-NEXT: and.b64 %rd14, %rd7, %rd1;
+; SM70-NEXT: and.b64 %rd14, %rd20, %rd1;
; SM70-NEXT: not.b64 %rd15, %rd14;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd20, [%rd9+16], %rd7, %rd15;
-; SM70-NEXT: setp.ne.b64 %p3, %rd20, %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
+; SM70-NEXT: mov.b64 %rd20, %rd7;
; SM70-NEXT: @%p3 bra $L__BB78_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
-; SM70-NEXT: $L__BB78_7: // %atomicrmw.start28
+; SM70-NEXT: $L__BB78_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b64 %rd16, %rd21, %rd6;
+; SM70-NEXT: and.b64 %rd16, %rd21, %rd2;
; SM70-NEXT: not.b64 %rd17, %rd16;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
; SM70-NEXT: mov.b64 %rd21, %rd8;
; SM70-NEXT: @%p4 bra $L__BB78_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end27
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd20, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
@@ -2757,92 +2757,92 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd17, [nand_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [nand_acq_rel_v8i64_global_cta_param_1+48];
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [nand_acq_rel_v8i64_global_cta_param_1+32];
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [nand_acq_rel_v8i64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [nand_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [nand_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [nand_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
; SM70-NEXT: $L__BB79_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd34;
-; SM70-NEXT: and.b64 %rd18, %rd5, %rd4;
+; SM70-NEXT: mov.b64 %rd7, %rd34;
+; SM70-NEXT: and.b64 %rd18, %rd7, %rd5;
; SM70-NEXT: not.b64 %rd19, %rd18;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd5, %rd19;
-; SM70-NEXT: setp.ne.b64 %p1, %rd34, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM70-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
; SM70-NEXT: @%p1 bra $L__BB79_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
-; SM70-NEXT: $L__BB79_3: // %atomicrmw.start12
+; SM70-NEXT: $L__BB79_3: // %atomicrmw.start10
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd6, %rd35;
-; SM70-NEXT: and.b64 %rd20, %rd6, %rd3;
+; SM70-NEXT: mov.b64 %rd8, %rd35;
+; SM70-NEXT: and.b64 %rd20, %rd8, %rd6;
; SM70-NEXT: not.b64 %rd21, %rd20;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd6, %rd21;
-; SM70-NEXT: setp.ne.b64 %p2, %rd35, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM70-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
; SM70-NEXT: @%p2 bra $L__BB79_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end11
+; SM70-NEXT: // %bb.4: // %atomicrmw.end9
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
-; SM70-NEXT: $L__BB79_5: // %atomicrmw.start24
+; SM70-NEXT: $L__BB79_5: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd8, %rd36;
-; SM70-NEXT: and.b64 %rd22, %rd8, %rd2;
+; SM70-NEXT: mov.b64 %rd9, %rd36;
+; SM70-NEXT: and.b64 %rd22, %rd9, %rd3;
; SM70-NEXT: not.b64 %rd23, %rd22;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd8, %rd23;
-; SM70-NEXT: setp.ne.b64 %p3, %rd36, %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM70-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
; SM70-NEXT: @%p3 bra $L__BB79_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end23
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
-; SM70-NEXT: $L__BB79_7: // %atomicrmw.start32
+; SM70-NEXT: $L__BB79_7: // %atomicrmw.start26
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd9, %rd37;
-; SM70-NEXT: and.b64 %rd24, %rd9, %rd7;
+; SM70-NEXT: mov.b64 %rd10, %rd37;
+; SM70-NEXT: and.b64 %rd24, %rd10, %rd4;
; SM70-NEXT: not.b64 %rd25, %rd24;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd9, %rd25;
-; SM70-NEXT: setp.ne.b64 %p4, %rd37, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM70-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
; SM70-NEXT: @%p4 bra $L__BB79_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end31
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
-; SM70-NEXT: $L__BB79_9: // %atomicrmw.start48
+; SM70-NEXT: $L__BB79_9: // %atomicrmw.start40
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd12, %rd38;
-; SM70-NEXT: and.b64 %rd26, %rd12, %rd1;
+; SM70-NEXT: mov.b64 %rd13, %rd38;
+; SM70-NEXT: and.b64 %rd26, %rd13, %rd1;
; SM70-NEXT: not.b64 %rd27, %rd26;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd12, %rd27;
-; SM70-NEXT: setp.ne.b64 %p5, %rd38, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM70-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
; SM70-NEXT: @%p5 bra $L__BB79_9;
-; SM70-NEXT: // %bb.10: // %atomicrmw.end47
+; SM70-NEXT: // %bb.10: // %atomicrmw.end39
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
-; SM70-NEXT: $L__BB79_11: // %atomicrmw.start56
+; SM70-NEXT: $L__BB79_11: // %atomicrmw.start46
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd13, %rd39;
-; SM70-NEXT: and.b64 %rd28, %rd13, %rd11;
+; SM70-NEXT: mov.b64 %rd14, %rd39;
+; SM70-NEXT: and.b64 %rd28, %rd14, %rd2;
; SM70-NEXT: not.b64 %rd29, %rd28;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd13, %rd29;
-; SM70-NEXT: setp.ne.b64 %p6, %rd39, %rd13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM70-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
; SM70-NEXT: @%p6 bra $L__BB79_11;
-; SM70-NEXT: // %bb.12: // %atomicrmw.end55
+; SM70-NEXT: // %bb.12: // %atomicrmw.end45
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
-; SM70-NEXT: $L__BB79_13: // %atomicrmw.start68
+; SM70-NEXT: $L__BB79_13: // %atomicrmw.start56
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd15, %rd40;
-; SM70-NEXT: and.b64 %rd30, %rd15, %rd10;
+; SM70-NEXT: and.b64 %rd30, %rd40, %rd11;
; SM70-NEXT: not.b64 %rd31, %rd30;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd40, [%rd17+48], %rd15, %rd31;
-; SM70-NEXT: setp.ne.b64 %p7, %rd40, %rd15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
+; SM70-NEXT: mov.b64 %rd40, %rd15;
; SM70-NEXT: @%p7 bra $L__BB79_13;
-; SM70-NEXT: // %bb.14: // %atomicrmw.end67
+; SM70-NEXT: // %bb.14: // %atomicrmw.end55
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
-; SM70-NEXT: $L__BB79_15: // %atomicrmw.start76
+; SM70-NEXT: $L__BB79_15: // %atomicrmw.start62
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b64 %rd32, %rd41, %rd14;
+; SM70-NEXT: and.b64 %rd32, %rd41, %rd12;
; SM70-NEXT: not.b64 %rd33, %rd32;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
; SM70-NEXT: mov.b64 %rd41, %rd16;
; SM70-NEXT: @%p8 bra $L__BB79_15;
-; SM70-NEXT: // %bb.16: // %atomicrmw.end75
+; SM70-NEXT: // %bb.16: // %atomicrmw.end61
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd40, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
@@ -2860,14 +2860,14 @@ define <1 x i8> @or_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %va
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b8 %r1, [or_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd2, %rd1, -4;
-; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
-; SM70-NEXT: and.b32 %r3, %r2, 3;
-; SM70-NEXT: shl.b32 %r4, %r3, 3;
-; SM70-NEXT: shl.b32 %r5, %r1, %r4;
+; SM70-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM70-NEXT: and.b32 %r2, %r1, 3;
+; SM70-NEXT: shl.b32 %r3, %r2, 3;
+; SM70-NEXT: ld.param.b8 %r4, [or_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r5, %r4, %r3;
; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
-; SM70-NEXT: shr.u32 %r7, %r6, %r4;
+; SM70-NEXT: shr.u32 %r7, %r6, %r3;
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b8 [func_retval0], %r7;
; SM70-NEXT: ret;
@@ -3123,10 +3123,8 @@ define <1 x i32> @or_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [or_acq_rel_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r2, [%rd1], %r1;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.or.b32 %r2, [%rd1], %r1;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -3207,10 +3205,8 @@ define <1 x i64> @or_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd2, [or_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd3, [%rd1], %rd2;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.or.b64 %rd3, [%rd1], %rd2;
; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -3296,14 +3292,14 @@ define <1 x i8> @xor_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b8 %r1, [xor_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd2, %rd1, -4;
-; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
-; SM70-NEXT: and.b32 %r3, %r2, 3;
-; SM70-NEXT: shl.b32 %r4, %r3, 3;
-; SM70-NEXT: shl.b32 %r5, %r1, %r4;
+; SM70-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM70-NEXT: and.b32 %r2, %r1, 3;
+; SM70-NEXT: shl.b32 %r3, %r2, 3;
+; SM70-NEXT: ld.param.b8 %r4, [xor_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r5, %r4, %r3;
; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
-; SM70-NEXT: shr.u32 %r7, %r6, %r4;
+; SM70-NEXT: shr.u32 %r7, %r6, %r3;
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b8 [func_retval0], %r7;
; SM70-NEXT: ret;
@@ -3559,10 +3555,8 @@ define <1 x i32> @xor_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [xor_acq_rel_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r2, [%rd1], %r1;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.xor.b32 %r2, [%rd1], %r1;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -3643,10 +3637,8 @@ define <1 x i64> @xor_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd2, [xor_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd3, [%rd1], %rd2;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.xor.b64 %rd3, [%rd1], %rd2;
; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -3727,14 +3719,14 @@ define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM70-LABEL: max_acq_rel_v1i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b16 %rs<5>;
; SM70-NEXT: .reg .b32 %r<15>;
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.s8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NEXT: and.b32 %r5, %r4, 3;
@@ -3742,12 +3734,13 @@ define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM70-NEXT: shl.b32 %r6, 255, %r1;
; SM70-NEXT: not.b32 %r2, %r6;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: cvt.s16.s8 %rs3, %rs1;
; SM70-NEXT: $L__BB112_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r7, %r14, %r1;
; SM70-NEXT: cvt.s8.s32 %rs2, %r7;
-; SM70-NEXT: max.s16 %rs3, %rs2, %rs1;
-; SM70-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM70-NEXT: max.s16 %rs4, %rs2, %rs3;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs4;
; SM70-NEXT: and.b32 %r9, %r8, 255;
; SM70-NEXT: shl.b32 %r10, %r9, %r1;
; SM70-NEXT: and.b32 %r11, %r14, %r2;
@@ -3994,9 +3987,9 @@ define <1 x i16> @max_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NEXT: and.b32 %r5, %r4, 3;
@@ -4191,10 +4184,8 @@ define <1 x i32> @max_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [max_acq_rel_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r2, [%rd1], %r1;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.max.s32 %r2, [%rd1], %r1;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -4275,10 +4266,8 @@ define <1 x i64> @max_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd3, [%rd1], %rd2;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.max.s64 %rd3, [%rd1], %rd2;
; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -4359,14 +4348,14 @@ define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM70-LABEL: min_acq_rel_v1i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b16 %rs<5>;
; SM70-NEXT: .reg .b32 %r<15>;
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.s8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NEXT: and.b32 %r5, %r4, 3;
@@ -4374,12 +4363,13 @@ define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM70-NEXT: shl.b32 %r6, 255, %r1;
; SM70-NEXT: not.b32 %r2, %r6;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: cvt.s16.s8 %rs3, %rs1;
; SM70-NEXT: $L__BB128_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r7, %r14, %r1;
; SM70-NEXT: cvt.s8.s32 %rs2, %r7;
-; SM70-NEXT: min.s16 %rs3, %rs2, %rs1;
-; SM70-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM70-NEXT: min.s16 %rs4, %rs2, %rs3;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs4;
; SM70-NEXT: and.b32 %r9, %r8, 255;
; SM70-NEXT: shl.b32 %r10, %r9, %r1;
; SM70-NEXT: and.b32 %r11, %r14, %r2;
@@ -4626,9 +4616,9 @@ define <1 x i16> @min_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NEXT: and.b32 %r5, %r4, 3;
@@ -4823,10 +4813,8 @@ define <1 x i32> @min_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [min_acq_rel_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r2, [%rd1], %r1;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.min.s32 %r2, [%rd1], %r1;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -4907,10 +4895,8 @@ define <1 x i64> @min_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd3, [%rd1], %rd2;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.min.s64 %rd3, [%rd1], %rd2;
; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -5233,9 +5219,9 @@ define <1 x i16> @umax_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NEXT: and.b32 %r5, %r4, 3;
@@ -5430,10 +5416,8 @@ define <1 x i32> @umax_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [umax_acq_rel_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r2, [%rd1], %r1;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.max.u32 %r2, [%rd1], %r1;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -5514,10 +5498,8 @@ define <1 x i64> @umax_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd3, [%rd1], %rd2;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.max.u64 %rd3, [%rd1], %rd2;
; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -5840,9 +5822,9 @@ define <1 x i16> @umin_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NEXT: and.b32 %r5, %r4, 3;
@@ -6037,10 +6019,8 @@ define <1 x i32> @umin_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [umin_acq_rel_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r2, [%rd1], %r1;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.min.u32 %r2, [%rd1], %r1;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -6121,10 +6101,8 @@ define <1 x i64> @umin_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd3, [%rd1], %rd2;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.min.u64 %rd3, [%rd1], %rd2;
; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -6466,9 +6444,9 @@ define <1 x i16> @uinc_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NEXT: and.b32 %r5, %r4, 3;
@@ -6693,10 +6671,8 @@ define <1 x i32> @uinc_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [uinc_wrap_acq_rel_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r2, [%rd1], %r1;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.inc.u32 %r2, [%rd1], %r1;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -6777,22 +6753,22 @@ define <1 x i64> @uinc_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM70-NEXT: .reg .b64 %rd<7>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd2];
; SM70-NEXT: $L__BB188_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd4, %rd6, 1;
-; SM70-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM70-NEXT: setp.ge.u64 %p1, %rd6, %rd3;
; SM70-NEXT: selp.b64 %rd5, 0, %rd4, %p1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
-; SM70-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
-; SM70-NEXT: mov.b64 %rd6, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5;
+; SM70-NEXT: setp.ne.b64 %p2, %rd1, %rd6;
+; SM70-NEXT: mov.b64 %rd6, %rd1;
; SM70-NEXT: @%p2 bra $L__BB188_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -6807,31 +6783,31 @@ define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd5, [uinc_wrap_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
; SM70-NEXT: $L__BB189_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd3, %rd10;
-; SM70-NEXT: add.s64 %rd6, %rd3, 1;
-; SM70-NEXT: setp.ge.u64 %p1, %rd3, %rd2;
+; SM70-NEXT: add.s64 %rd6, %rd10, 1;
+; SM70-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
; SM70-NEXT: selp.b64 %rd7, 0, %rd6, %p1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd10, [%rd5], %rd3, %rd7;
-; SM70-NEXT: setp.ne.b64 %p2, %rd10, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM70-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
+; SM70-NEXT: mov.b64 %rd10, %rd3;
; SM70-NEXT: @%p2 bra $L__BB189_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
-; SM70-NEXT: $L__BB189_3: // %atomicrmw.start4
+; SM70-NEXT: $L__BB189_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd8, %rd11, 1;
-; SM70-NEXT: setp.ge.u64 %p3, %rd11, %rd1;
+; SM70-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
; SM70-NEXT: selp.b64 %rd9, 0, %rd8, %p3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM70-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
; SM70-NEXT: mov.b64 %rd11, %rd4;
; SM70-NEXT: @%p4 bra $L__BB189_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end3
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd4};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -6846,54 +6822,54 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd9, [uinc_wrap_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
; SM70-NEXT: $L__BB190_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd4, %rd18;
-; SM70-NEXT: add.s64 %rd10, %rd4, 1;
-; SM70-NEXT: setp.ge.u64 %p1, %rd4, %rd3;
+; SM70-NEXT: mov.b64 %rd5, %rd18;
+; SM70-NEXT: add.s64 %rd10, %rd5, 1;
+; SM70-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
; SM70-NEXT: selp.b64 %rd11, 0, %rd10, %p1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd4, %rd11;
-; SM70-NEXT: setp.ne.b64 %p2, %rd18, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
; SM70-NEXT: @%p2 bra $L__BB190_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
-; SM70-NEXT: $L__BB190_3: // %atomicrmw.start8
+; SM70-NEXT: $L__BB190_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd19;
-; SM70-NEXT: add.s64 %rd12, %rd5, 1;
-; SM70-NEXT: setp.ge.u64 %p3, %rd5, %rd2;
+; SM70-NEXT: mov.b64 %rd6, %rd19;
+; SM70-NEXT: add.s64 %rd12, %rd6, 1;
+; SM70-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
; SM70-NEXT: selp.b64 %rd13, 0, %rd12, %p3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd5, %rd13;
-; SM70-NEXT: setp.ne.b64 %p4, %rd19, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
; SM70-NEXT: @%p4 bra $L__BB190_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end7
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
-; SM70-NEXT: $L__BB190_5: // %atomicrmw.start20
+; SM70-NEXT: $L__BB190_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd7, %rd20;
-; SM70-NEXT: add.s64 %rd14, %rd7, 1;
-; SM70-NEXT: setp.ge.u64 %p5, %rd7, %rd1;
+; SM70-NEXT: add.s64 %rd14, %rd20, 1;
+; SM70-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
; SM70-NEXT: selp.b64 %rd15, 0, %rd14, %p5;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd20, [%rd9+16], %rd7, %rd15;
-; SM70-NEXT: setp.ne.b64 %p6, %rd20, %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM70-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
+; SM70-NEXT: mov.b64 %rd20, %rd7;
; SM70-NEXT: @%p6 bra $L__BB190_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
-; SM70-NEXT: $L__BB190_7: // %atomicrmw.start28
+; SM70-NEXT: $L__BB190_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd16, %rd21, 1;
-; SM70-NEXT: setp.ge.u64 %p7, %rd21, %rd6;
+; SM70-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
; SM70-NEXT: selp.b64 %rd17, 0, %rd16, %p7;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM70-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
; SM70-NEXT: mov.b64 %rd21, %rd8;
; SM70-NEXT: @%p8 bra $L__BB190_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end27
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd20, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
@@ -6909,100 +6885,100 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd17, [uinc_wrap_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+48];
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+32];
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
; SM70-NEXT: $L__BB191_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd34;
-; SM70-NEXT: add.s64 %rd18, %rd5, 1;
-; SM70-NEXT: setp.ge.u64 %p1, %rd5, %rd4;
+; SM70-NEXT: mov.b64 %rd7, %rd34;
+; SM70-NEXT: add.s64 %rd18, %rd7, 1;
+; SM70-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
; SM70-NEXT: selp.b64 %rd19, 0, %rd18, %p1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd5, %rd19;
-; SM70-NEXT: setp.ne.b64 %p2, %rd34, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
; SM70-NEXT: @%p2 bra $L__BB191_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
-; SM70-NEXT: $L__BB191_3: // %atomicrmw.start12
+; SM70-NEXT: $L__BB191_3: // %atomicrmw.start10
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd6, %rd35;
-; SM70-NEXT: add.s64 %rd20, %rd6, 1;
-; SM70-NEXT: setp.ge.u64 %p3, %rd6, %rd3;
+; SM70-NEXT: mov.b64 %rd8, %rd35;
+; SM70-NEXT: add.s64 %rd20, %rd8, 1;
+; SM70-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
; SM70-NEXT: selp.b64 %rd21, 0, %rd20, %p3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd6, %rd21;
-; SM70-NEXT: setp.ne.b64 %p4, %rd35, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
; SM70-NEXT: @%p4 bra $L__BB191_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end11
+; SM70-NEXT: // %bb.4: // %atomicrmw.end9
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
-; SM70-NEXT: $L__BB191_5: // %atomicrmw.start24
+; SM70-NEXT: $L__BB191_5: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd8, %rd36;
-; SM70-NEXT: add.s64 %rd22, %rd8, 1;
-; SM70-NEXT: setp.ge.u64 %p5, %rd8, %rd2;
+; SM70-NEXT: mov.b64 %rd9, %rd36;
+; SM70-NEXT: add.s64 %rd22, %rd9, 1;
+; SM70-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
; SM70-NEXT: selp.b64 %rd23, 0, %rd22, %p5;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd8, %rd23;
-; SM70-NEXT: setp.ne.b64 %p6, %rd36, %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
; SM70-NEXT: @%p6 bra $L__BB191_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end23
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
-; SM70-NEXT: $L__BB191_7: // %atomicrmw.start32
+; SM70-NEXT: $L__BB191_7: // %atomicrmw.start26
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd9, %rd37;
-; SM70-NEXT: add.s64 %rd24, %rd9, 1;
-; SM70-NEXT: setp.ge.u64 %p7, %rd9, %rd7;
+; SM70-NEXT: mov.b64 %rd10, %rd37;
+; SM70-NEXT: add.s64 %rd24, %rd10, 1;
+; SM70-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
; SM70-NEXT: selp.b64 %rd25, 0, %rd24, %p7;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd9, %rd25;
-; SM70-NEXT: setp.ne.b64 %p8, %rd37, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
; SM70-NEXT: @%p8 bra $L__BB191_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end31
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
-; SM70-NEXT: $L__BB191_9: // %atomicrmw.start48
+; SM70-NEXT: $L__BB191_9: // %atomicrmw.start40
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd12, %rd38;
-; SM70-NEXT: add.s64 %rd26, %rd12, 1;
-; SM70-NEXT: setp.ge.u64 %p9, %rd12, %rd1;
+; SM70-NEXT: mov.b64 %rd13, %rd38;
+; SM70-NEXT: add.s64 %rd26, %rd13, 1;
+; SM70-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
; SM70-NEXT: selp.b64 %rd27, 0, %rd26, %p9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd12, %rd27;
-; SM70-NEXT: setp.ne.b64 %p10, %rd38, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM70-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
; SM70-NEXT: @%p10 bra $L__BB191_9;
-; SM70-NEXT: // %bb.10: // %atomicrmw.end47
+; SM70-NEXT: // %bb.10: // %atomicrmw.end39
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
-; SM70-NEXT: $L__BB191_11: // %atomicrmw.start56
+; SM70-NEXT: $L__BB191_11: // %atomicrmw.start46
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd13, %rd39;
-; SM70-NEXT: add.s64 %rd28, %rd13, 1;
-; SM70-NEXT: setp.ge.u64 %p11, %rd13, %rd11;
+; SM70-NEXT: mov.b64 %rd14, %rd39;
+; SM70-NEXT: add.s64 %rd28, %rd14, 1;
+; SM70-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
; SM70-NEXT: selp.b64 %rd29, 0, %rd28, %p11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd13, %rd29;
-; SM70-NEXT: setp.ne.b64 %p12, %rd39, %rd13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM70-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
; SM70-NEXT: @%p12 bra $L__BB191_11;
-; SM70-NEXT: // %bb.12: // %atomicrmw.end55
+; SM70-NEXT: // %bb.12: // %atomicrmw.end45
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
-; SM70-NEXT: $L__BB191_13: // %atomicrmw.start68
+; SM70-NEXT: $L__BB191_13: // %atomicrmw.start56
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd15, %rd40;
-; SM70-NEXT: add.s64 %rd30, %rd15, 1;
-; SM70-NEXT: setp.ge.u64 %p13, %rd15, %rd10;
+; SM70-NEXT: add.s64 %rd30, %rd40, 1;
+; SM70-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
; SM70-NEXT: selp.b64 %rd31, 0, %rd30, %p13;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd40, [%rd17+48], %rd15, %rd31;
-; SM70-NEXT: setp.ne.b64 %p14, %rd40, %rd15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM70-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
+; SM70-NEXT: mov.b64 %rd40, %rd15;
; SM70-NEXT: @%p14 bra $L__BB191_13;
-; SM70-NEXT: // %bb.14: // %atomicrmw.end67
+; SM70-NEXT: // %bb.14: // %atomicrmw.end55
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
-; SM70-NEXT: $L__BB191_15: // %atomicrmw.start76
+; SM70-NEXT: $L__BB191_15: // %atomicrmw.start62
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd32, %rd41, 1;
-; SM70-NEXT: setp.ge.u64 %p15, %rd41, %rd14;
+; SM70-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
; SM70-NEXT: selp.b64 %rd33, 0, %rd32, %p15;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM70-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
; SM70-NEXT: mov.b64 %rd41, %rd16;
; SM70-NEXT: @%p16 bra $L__BB191_15;
-; SM70-NEXT: // %bb.16: // %atomicrmw.end75
+; SM70-NEXT: // %bb.16: // %atomicrmw.end61
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd40, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
@@ -7314,9 +7290,9 @@ define <1 x i16> @udec_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NEXT: and.b32 %r5, %r4, 3;
@@ -7571,10 +7547,8 @@ define <1 x i32> @udec_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [udec_wrap_acq_rel_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r2, [%rd1], %r1;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.dec.u32 %r2, [%rd1], %r1;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -7655,24 +7629,24 @@ define <1 x i64> @udec_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM70-NEXT: .reg .b64 %rd<8>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd7, [%rd3];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd7, [%rd2];
; SM70-NEXT: $L__BB204_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd4, %rd7, -1;
; SM70-NEXT: setp.eq.b64 %p1, %rd7, 0;
-; SM70-NEXT: setp.gt.u64 %p2, %rd7, %rd1;
-; SM70-NEXT: selp.b64 %rd5, %rd1, %rd4, %p2;
-; SM70-NEXT: selp.b64 %rd6, %rd1, %rd5, %p1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd7, %rd6;
-; SM70-NEXT: setp.ne.b64 %p3, %rd2, %rd7;
-; SM70-NEXT: mov.b64 %rd7, %rd2;
+; SM70-NEXT: setp.gt.u64 %p2, %rd7, %rd3;
+; SM70-NEXT: selp.b64 %rd5, %rd3, %rd4, %p2;
+; SM70-NEXT: selp.b64 %rd6, %rd3, %rd5, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd7, %rd6;
+; SM70-NEXT: setp.ne.b64 %p3, %rd1, %rd7;
+; SM70-NEXT: mov.b64 %rd7, %rd1;
; SM70-NEXT: @%p3 bra $L__BB204_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -7687,35 +7661,35 @@ define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd5, [udec_wrap_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd5];
; SM70-NEXT: $L__BB205_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd3, %rd12;
-; SM70-NEXT: add.s64 %rd6, %rd3, -1;
-; SM70-NEXT: setp.eq.b64 %p1, %rd3, 0;
-; SM70-NEXT: setp.gt.u64 %p2, %rd3, %rd2;
-; SM70-NEXT: selp.b64 %rd7, %rd2, %rd6, %p2;
-; SM70-NEXT: selp.b64 %rd8, %rd2, %rd7, %p1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd5], %rd3, %rd8;
-; SM70-NEXT: setp.ne.b64 %p3, %rd12, %rd3;
+; SM70-NEXT: add.s64 %rd6, %rd12, -1;
+; SM70-NEXT: setp.eq.b64 %p1, %rd12, 0;
+; SM70-NEXT: setp.gt.u64 %p2, %rd12, %rd1;
+; SM70-NEXT: selp.b64 %rd7, %rd1, %rd6, %p2;
+; SM70-NEXT: selp.b64 %rd8, %rd1, %rd7, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd12, %rd8;
+; SM70-NEXT: setp.ne.b64 %p3, %rd3, %rd12;
+; SM70-NEXT: mov.b64 %rd12, %rd3;
; SM70-NEXT: @%p3 bra $L__BB205_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd5+8];
-; SM70-NEXT: $L__BB205_3: // %atomicrmw.start4
+; SM70-NEXT: $L__BB205_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd9, %rd13, -1;
; SM70-NEXT: setp.eq.b64 %p4, %rd13, 0;
-; SM70-NEXT: setp.gt.u64 %p5, %rd13, %rd1;
-; SM70-NEXT: selp.b64 %rd10, %rd1, %rd9, %p5;
-; SM70-NEXT: selp.b64 %rd11, %rd1, %rd10, %p4;
+; SM70-NEXT: setp.gt.u64 %p5, %rd13, %rd2;
+; SM70-NEXT: selp.b64 %rd10, %rd2, %rd9, %p5;
+; SM70-NEXT: selp.b64 %rd11, %rd2, %rd10, %p4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd13, %rd11;
; SM70-NEXT: setp.ne.b64 %p6, %rd4, %rd13;
; SM70-NEXT: mov.b64 %rd13, %rd4;
; SM70-NEXT: @%p6 bra $L__BB205_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end3
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd12, %rd4};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -7730,62 +7704,62 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd9, [udec_wrap_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd9];
; SM70-NEXT: $L__BB206_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd4, %rd22;
-; SM70-NEXT: add.s64 %rd10, %rd4, -1;
-; SM70-NEXT: setp.eq.b64 %p1, %rd4, 0;
-; SM70-NEXT: setp.gt.u64 %p2, %rd4, %rd3;
+; SM70-NEXT: mov.b64 %rd5, %rd22;
+; SM70-NEXT: add.s64 %rd10, %rd5, -1;
+; SM70-NEXT: setp.eq.b64 %p1, %rd5, 0;
+; SM70-NEXT: setp.gt.u64 %p2, %rd5, %rd3;
; SM70-NEXT: selp.b64 %rd11, %rd3, %rd10, %p2;
; SM70-NEXT: selp.b64 %rd12, %rd3, %rd11, %p1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd22, [%rd9], %rd4, %rd12;
-; SM70-NEXT: setp.ne.b64 %p3, %rd22, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd22, [%rd9], %rd5, %rd12;
+; SM70-NEXT: setp.ne.b64 %p3, %rd22, %rd5;
; SM70-NEXT: @%p3 bra $L__BB206_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd23, [%rd9+8];
-; SM70-NEXT: $L__BB206_3: // %atomicrmw.start8
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd23;
-; SM70-NEXT: add.s64 %rd13, %rd5, -1;
-; SM70-NEXT: setp.eq.b64 %p4, %rd5, 0;
-; SM70-NEXT: setp.gt.u64 %p5, %rd5, %rd2;
-; SM70-NEXT: selp.b64 %rd14, %rd2, %rd13, %p5;
-; SM70-NEXT: selp.b64 %rd15, %rd2, %rd14, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd9+8], %rd5, %rd15;
-; SM70-NEXT: setp.ne.b64 %p6, %rd23, %rd5;
+; SM70-NEXT: $L__BB206_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd23;
+; SM70-NEXT: add.s64 %rd13, %rd6, -1;
+; SM70-NEXT: setp.eq.b64 %p4, %rd6, 0;
+; SM70-NEXT: setp.gt.u64 %p5, %rd6, %rd4;
+; SM70-NEXT: selp.b64 %rd14, %rd4, %rd13, %p5;
+; SM70-NEXT: selp.b64 %rd15, %rd4, %rd14, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd9+8], %rd6, %rd15;
+; SM70-NEXT: setp.ne.b64 %p6, %rd23, %rd6;
; SM70-NEXT: @%p6 bra $L__BB206_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end7
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd9+16];
-; SM70-NEXT: $L__BB206_5: // %atomicrmw.start20
+; SM70-NEXT: $L__BB206_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd7, %rd24;
-; SM70-NEXT: add.s64 %rd16, %rd7, -1;
-; SM70-NEXT: setp.eq.b64 %p7, %rd7, 0;
-; SM70-NEXT: setp.gt.u64 %p8, %rd7, %rd1;
+; SM70-NEXT: add.s64 %rd16, %rd24, -1;
+; SM70-NEXT: setp.eq.b64 %p7, %rd24, 0;
+; SM70-NEXT: setp.gt.u64 %p8, %rd24, %rd1;
; SM70-NEXT: selp.b64 %rd17, %rd1, %rd16, %p8;
; SM70-NEXT: selp.b64 %rd18, %rd1, %rd17, %p7;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd24, [%rd9+16], %rd7, %rd18;
-; SM70-NEXT: setp.ne.b64 %p9, %rd24, %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd24, %rd18;
+; SM70-NEXT: setp.ne.b64 %p9, %rd7, %rd24;
+; SM70-NEXT: mov.b64 %rd24, %rd7;
; SM70-NEXT: @%p9 bra $L__BB206_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd25, [%rd9+24];
-; SM70-NEXT: $L__BB206_7: // %atomicrmw.start28
+; SM70-NEXT: $L__BB206_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd19, %rd25, -1;
; SM70-NEXT: setp.eq.b64 %p10, %rd25, 0;
-; SM70-NEXT: setp.gt.u64 %p11, %rd25, %rd6;
-; SM70-NEXT: selp.b64 %rd20, %rd6, %rd19, %p11;
-; SM70-NEXT: selp.b64 %rd21, %rd6, %rd20, %p10;
+; SM70-NEXT: setp.gt.u64 %p11, %rd25, %rd2;
+; SM70-NEXT: selp.b64 %rd20, %rd2, %rd19, %p11;
+; SM70-NEXT: selp.b64 %rd21, %rd2, %rd20, %p10;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd25, %rd21;
; SM70-NEXT: setp.ne.b64 %p12, %rd8, %rd25;
; SM70-NEXT: mov.b64 %rd25, %rd8;
; SM70-NEXT: @%p12 bra $L__BB206_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end27
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd24, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd22, %rd23};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
@@ -7801,116 +7775,116 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd17, [udec_wrap_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+48];
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+32];
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd42, [%rd17];
; SM70-NEXT: $L__BB207_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd42;
-; SM70-NEXT: add.s64 %rd18, %rd5, -1;
-; SM70-NEXT: setp.eq.b64 %p1, %rd5, 0;
-; SM70-NEXT: setp.gt.u64 %p2, %rd5, %rd4;
-; SM70-NEXT: selp.b64 %rd19, %rd4, %rd18, %p2;
-; SM70-NEXT: selp.b64 %rd20, %rd4, %rd19, %p1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd42, [%rd17], %rd5, %rd20;
-; SM70-NEXT: setp.ne.b64 %p3, %rd42, %rd5;
+; SM70-NEXT: mov.b64 %rd7, %rd42;
+; SM70-NEXT: add.s64 %rd18, %rd7, -1;
+; SM70-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM70-NEXT: setp.gt.u64 %p2, %rd7, %rd5;
+; SM70-NEXT: selp.b64 %rd19, %rd5, %rd18, %p2;
+; SM70-NEXT: selp.b64 %rd20, %rd5, %rd19, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd42, [%rd17], %rd7, %rd20;
+; SM70-NEXT: setp.ne.b64 %p3, %rd42, %rd7;
; SM70-NEXT: @%p3 bra $L__BB207_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd43, [%rd17+8];
-; SM70-NEXT: $L__BB207_3: // %atomicrmw.start12
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd6, %rd43;
-; SM70-NEXT: add.s64 %rd21, %rd6, -1;
-; SM70-NEXT: setp.eq.b64 %p4, %rd6, 0;
-; SM70-NEXT: setp.gt.u64 %p5, %rd6, %rd3;
-; SM70-NEXT: selp.b64 %rd22, %rd3, %rd21, %p5;
-; SM70-NEXT: selp.b64 %rd23, %rd3, %rd22, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd43, [%rd17+8], %rd6, %rd23;
-; SM70-NEXT: setp.ne.b64 %p6, %rd43, %rd6;
+; SM70-NEXT: $L__BB207_3: // %atomicrmw.start10
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd43;
+; SM70-NEXT: add.s64 %rd21, %rd8, -1;
+; SM70-NEXT: setp.eq.b64 %p4, %rd8, 0;
+; SM70-NEXT: setp.gt.u64 %p5, %rd8, %rd6;
+; SM70-NEXT: selp.b64 %rd22, %rd6, %rd21, %p5;
+; SM70-NEXT: selp.b64 %rd23, %rd6, %rd22, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd43, [%rd17+8], %rd8, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd43, %rd8;
; SM70-NEXT: @%p6 bra $L__BB207_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end11
+; SM70-NEXT: // %bb.4: // %atomicrmw.end9
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd44, [%rd17+16];
-; SM70-NEXT: $L__BB207_5: // %atomicrmw.start24
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd8, %rd44;
-; SM70-NEXT: add.s64 %rd24, %rd8, -1;
-; SM70-NEXT: setp.eq.b64 %p7, %rd8, 0;
-; SM70-NEXT: setp.gt.u64 %p8, %rd8, %rd2;
-; SM70-NEXT: selp.b64 %rd25, %rd2, %rd24, %p8;
-; SM70-NEXT: selp.b64 %rd26, %rd2, %rd25, %p7;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd44, [%rd17+16], %rd8, %rd26;
-; SM70-NEXT: setp.ne.b64 %p9, %rd44, %rd8;
+; SM70-NEXT: $L__BB207_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd44;
+; SM70-NEXT: add.s64 %rd24, %rd9, -1;
+; SM70-NEXT: setp.eq.b64 %p7, %rd9, 0;
+; SM70-NEXT: setp.gt.u64 %p8, %rd9, %rd3;
+; SM70-NEXT: selp.b64 %rd25, %rd3, %rd24, %p8;
+; SM70-NEXT: selp.b64 %rd26, %rd3, %rd25, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd44, [%rd17+16], %rd9, %rd26;
+; SM70-NEXT: setp.ne.b64 %p9, %rd44, %rd9;
; SM70-NEXT: @%p9 bra $L__BB207_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end23
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd45, [%rd17+24];
-; SM70-NEXT: $L__BB207_7: // %atomicrmw.start32
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd9, %rd45;
-; SM70-NEXT: add.s64 %rd27, %rd9, -1;
-; SM70-NEXT: setp.eq.b64 %p10, %rd9, 0;
-; SM70-NEXT: setp.gt.u64 %p11, %rd9, %rd7;
-; SM70-NEXT: selp.b64 %rd28, %rd7, %rd27, %p11;
-; SM70-NEXT: selp.b64 %rd29, %rd7, %rd28, %p10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd17+24], %rd9, %rd29;
-; SM70-NEXT: setp.ne.b64 %p12, %rd45, %rd9;
+; SM70-NEXT: $L__BB207_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd45;
+; SM70-NEXT: add.s64 %rd27, %rd10, -1;
+; SM70-NEXT: setp.eq.b64 %p10, %rd10, 0;
+; SM70-NEXT: setp.gt.u64 %p11, %rd10, %rd4;
+; SM70-NEXT: selp.b64 %rd28, %rd4, %rd27, %p11;
+; SM70-NEXT: selp.b64 %rd29, %rd4, %rd28, %p10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd17+24], %rd10, %rd29;
+; SM70-NEXT: setp.ne.b64 %p12, %rd45, %rd10;
; SM70-NEXT: @%p12 bra $L__BB207_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end31
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd46, [%rd17+32];
-; SM70-NEXT: $L__BB207_9: // %atomicrmw.start48
+; SM70-NEXT: $L__BB207_9: // %atomicrmw.start40
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd12, %rd46;
-; SM70-NEXT: add.s64 %rd30, %rd12, -1;
-; SM70-NEXT: setp.eq.b64 %p13, %rd12, 0;
-; SM70-NEXT: setp.gt.u64 %p14, %rd12, %rd1;
+; SM70-NEXT: mov.b64 %rd13, %rd46;
+; SM70-NEXT: add.s64 %rd30, %rd13, -1;
+; SM70-NEXT: setp.eq.b64 %p13, %rd13, 0;
+; SM70-NEXT: setp.gt.u64 %p14, %rd13, %rd1;
; SM70-NEXT: selp.b64 %rd31, %rd1, %rd30, %p14;
; SM70-NEXT: selp.b64 %rd32, %rd1, %rd31, %p13;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd46, [%rd17+32], %rd12, %rd32;
-; SM70-NEXT: setp.ne.b64 %p15, %rd46, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd46, [%rd17+32], %rd13, %rd32;
+; SM70-NEXT: setp.ne.b64 %p15, %rd46, %rd13;
; SM70-NEXT: @%p15 bra $L__BB207_9;
-; SM70-NEXT: // %bb.10: // %atomicrmw.end47
+; SM70-NEXT: // %bb.10: // %atomicrmw.end39
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd47, [%rd17+40];
-; SM70-NEXT: $L__BB207_11: // %atomicrmw.start56
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd13, %rd47;
-; SM70-NEXT: add.s64 %rd33, %rd13, -1;
-; SM70-NEXT: setp.eq.b64 %p16, %rd13, 0;
-; SM70-NEXT: setp.gt.u64 %p17, %rd13, %rd11;
-; SM70-NEXT: selp.b64 %rd34, %rd11, %rd33, %p17;
-; SM70-NEXT: selp.b64 %rd35, %rd11, %rd34, %p16;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd47, [%rd17+40], %rd13, %rd35;
-; SM70-NEXT: setp.ne.b64 %p18, %rd47, %rd13;
+; SM70-NEXT: $L__BB207_11: // %atomicrmw.start46
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd47;
+; SM70-NEXT: add.s64 %rd33, %rd14, -1;
+; SM70-NEXT: setp.eq.b64 %p16, %rd14, 0;
+; SM70-NEXT: setp.gt.u64 %p17, %rd14, %rd2;
+; SM70-NEXT: selp.b64 %rd34, %rd2, %rd33, %p17;
+; SM70-NEXT: selp.b64 %rd35, %rd2, %rd34, %p16;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd47, [%rd17+40], %rd14, %rd35;
+; SM70-NEXT: setp.ne.b64 %p18, %rd47, %rd14;
; SM70-NEXT: @%p18 bra $L__BB207_11;
-; SM70-NEXT: // %bb.12: // %atomicrmw.end55
+; SM70-NEXT: // %bb.12: // %atomicrmw.end45
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd48, [%rd17+48];
-; SM70-NEXT: $L__BB207_13: // %atomicrmw.start68
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd15, %rd48;
-; SM70-NEXT: add.s64 %rd36, %rd15, -1;
-; SM70-NEXT: setp.eq.b64 %p19, %rd15, 0;
-; SM70-NEXT: setp.gt.u64 %p20, %rd15, %rd10;
-; SM70-NEXT: selp.b64 %rd37, %rd10, %rd36, %p20;
-; SM70-NEXT: selp.b64 %rd38, %rd10, %rd37, %p19;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd48, [%rd17+48], %rd15, %rd38;
-; SM70-NEXT: setp.ne.b64 %p21, %rd48, %rd15;
+; SM70-NEXT: $L__BB207_13: // %atomicrmw.start56
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd36, %rd48, -1;
+; SM70-NEXT: setp.eq.b64 %p19, %rd48, 0;
+; SM70-NEXT: setp.gt.u64 %p20, %rd48, %rd11;
+; SM70-NEXT: selp.b64 %rd37, %rd11, %rd36, %p20;
+; SM70-NEXT: selp.b64 %rd38, %rd11, %rd37, %p19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd48, %rd38;
+; SM70-NEXT: setp.ne.b64 %p21, %rd15, %rd48;
+; SM70-NEXT: mov.b64 %rd48, %rd15;
; SM70-NEXT: @%p21 bra $L__BB207_13;
-; SM70-NEXT: // %bb.14: // %atomicrmw.end67
+; SM70-NEXT: // %bb.14: // %atomicrmw.end55
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd49, [%rd17+56];
-; SM70-NEXT: $L__BB207_15: // %atomicrmw.start76
+; SM70-NEXT: $L__BB207_15: // %atomicrmw.start62
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd39, %rd49, -1;
; SM70-NEXT: setp.eq.b64 %p22, %rd49, 0;
-; SM70-NEXT: setp.gt.u64 %p23, %rd49, %rd14;
-; SM70-NEXT: selp.b64 %rd40, %rd14, %rd39, %p23;
-; SM70-NEXT: selp.b64 %rd41, %rd14, %rd40, %p22;
+; SM70-NEXT: setp.gt.u64 %p23, %rd49, %rd12;
+; SM70-NEXT: selp.b64 %rd40, %rd12, %rd39, %p23;
+; SM70-NEXT: selp.b64 %rd41, %rd12, %rd40, %p22;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd49, %rd41;
; SM70-NEXT: setp.ne.b64 %p24, %rd16, %rd49;
; SM70-NEXT: mov.b64 %rd49, %rd16;
; SM70-NEXT: @%p24 bra $L__BB207_15;
-; SM70-NEXT: // %bb.16: // %atomicrmw.end75
+; SM70-NEXT: // %bb.16: // %atomicrmw.end61
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd48, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd46, %rd47};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
@@ -8196,9 +8170,9 @@ define <1 x i16> @usub_cond_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NEXT: and.b32 %r5, %r4, 3;
@@ -8423,22 +8397,22 @@ define <1 x i32> @usub_cond_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b32 %r1, [usub_cond_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT: $L__BB216_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.ge.u32 %p1, %r5, %r1;
-; SM70-NEXT: sub.s32 %r3, %r5, %r1;
+; SM70-NEXT: setp.ge.u32 %p1, %r5, %r2;
+; SM70-NEXT: sub.s32 %r3, %r5, %r2;
; SM70-NEXT: selp.b32 %r4, %r3, %r5, %p1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM70-NEXT: setp.ne.b32 %p2, %r2, %r5;
-; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p2, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
; SM70-NEXT: @%p2 bra $L__BB216_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -8682,22 +8656,22 @@ define <1 x i64> @usub_cond_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM70-NEXT: .reg .b64 %rd<7>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd2];
; SM70-NEXT: $L__BB220_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
-; SM70-NEXT: sub.s64 %rd4, %rd6, %rd1;
+; SM70-NEXT: setp.ge.u64 %p1, %rd6, %rd3;
+; SM70-NEXT: sub.s64 %rd4, %rd6, %rd3;
; SM70-NEXT: selp.b64 %rd5, %rd4, %rd6, %p1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
-; SM70-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
-; SM70-NEXT: mov.b64 %rd6, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5;
+; SM70-NEXT: setp.ne.b64 %p2, %rd1, %rd6;
+; SM70-NEXT: mov.b64 %rd6, %rd1;
; SM70-NEXT: @%p2 bra $L__BB220_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -8712,31 +8686,31 @@ define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd5, [usub_cond_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
; SM70-NEXT: $L__BB221_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd3, %rd10;
-; SM70-NEXT: setp.ge.u64 %p1, %rd3, %rd2;
-; SM70-NEXT: sub.s64 %rd6, %rd3, %rd2;
-; SM70-NEXT: selp.b64 %rd7, %rd6, %rd3, %p1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd10, [%rd5], %rd3, %rd7;
-; SM70-NEXT: setp.ne.b64 %p2, %rd10, %rd3;
+; SM70-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
+; SM70-NEXT: sub.s64 %rd6, %rd10, %rd1;
+; SM70-NEXT: selp.b64 %rd7, %rd6, %rd10, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM70-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
+; SM70-NEXT: mov.b64 %rd10, %rd3;
; SM70-NEXT: @%p2 bra $L__BB221_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
-; SM70-NEXT: $L__BB221_3: // %atomicrmw.start4
+; SM70-NEXT: $L__BB221_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.ge.u64 %p3, %rd11, %rd1;
-; SM70-NEXT: sub.s64 %rd8, %rd11, %rd1;
+; SM70-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
+; SM70-NEXT: sub.s64 %rd8, %rd11, %rd2;
; SM70-NEXT: selp.b64 %rd9, %rd8, %rd11, %p3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM70-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
; SM70-NEXT: mov.b64 %rd11, %rd4;
; SM70-NEXT: @%p4 bra $L__BB221_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end3
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd4};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -8751,54 +8725,54 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd9, [usub_cond_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
; SM70-NEXT: $L__BB222_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd4, %rd18;
-; SM70-NEXT: setp.ge.u64 %p1, %rd4, %rd3;
-; SM70-NEXT: sub.s64 %rd10, %rd4, %rd3;
-; SM70-NEXT: selp.b64 %rd11, %rd10, %rd4, %p1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd4, %rd11;
-; SM70-NEXT: setp.ne.b64 %p2, %rd18, %rd4;
+; SM70-NEXT: mov.b64 %rd5, %rd18;
+; SM70-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
+; SM70-NEXT: sub.s64 %rd10, %rd5, %rd3;
+; SM70-NEXT: selp.b64 %rd11, %rd10, %rd5, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
; SM70-NEXT: @%p2 bra $L__BB222_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
-; SM70-NEXT: $L__BB222_3: // %atomicrmw.start8
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd19;
-; SM70-NEXT: setp.ge.u64 %p3, %rd5, %rd2;
-; SM70-NEXT: sub.s64 %rd12, %rd5, %rd2;
-; SM70-NEXT: selp.b64 %rd13, %rd12, %rd5, %p3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd5, %rd13;
-; SM70-NEXT: setp.ne.b64 %p4, %rd19, %rd5;
+; SM70-NEXT: $L__BB222_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd19;
+; SM70-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
+; SM70-NEXT: sub.s64 %rd12, %rd6, %rd4;
+; SM70-NEXT: selp.b64 %rd13, %rd12, %rd6, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
; SM70-NEXT: @%p4 bra $L__BB222_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end7
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
-; SM70-NEXT: $L__BB222_5: // %atomicrmw.start20
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd7, %rd20;
-; SM70-NEXT: setp.ge.u64 %p5, %rd7, %rd1;
-; SM70-NEXT: sub.s64 %rd14, %rd7, %rd1;
-; SM70-NEXT: selp.b64 %rd15, %rd14, %rd7, %p5;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd20, [%rd9+16], %rd7, %rd15;
-; SM70-NEXT: setp.ne.b64 %p6, %rd20, %rd7;
+; SM70-NEXT: $L__BB222_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
+; SM70-NEXT: sub.s64 %rd14, %rd20, %rd1;
+; SM70-NEXT: selp.b64 %rd15, %rd14, %rd20, %p5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM70-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
+; SM70-NEXT: mov.b64 %rd20, %rd7;
; SM70-NEXT: @%p6 bra $L__BB222_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
-; SM70-NEXT: $L__BB222_7: // %atomicrmw.start28
+; SM70-NEXT: $L__BB222_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.ge.u64 %p7, %rd21, %rd6;
-; SM70-NEXT: sub.s64 %rd16, %rd21, %rd6;
+; SM70-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
+; SM70-NEXT: sub.s64 %rd16, %rd21, %rd2;
; SM70-NEXT: selp.b64 %rd17, %rd16, %rd21, %p7;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM70-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
; SM70-NEXT: mov.b64 %rd21, %rd8;
; SM70-NEXT: @%p8 bra $L__BB222_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end27
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd20, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
@@ -8814,100 +8788,100 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd17, [usub_cond_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [usub_cond_acq_rel_v8i64_global_cta_param_1+48];
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [usub_cond_acq_rel_v8i64_global_cta_param_1+32];
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [usub_cond_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
; SM70-NEXT: $L__BB223_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd34;
-; SM70-NEXT: setp.ge.u64 %p1, %rd5, %rd4;
-; SM70-NEXT: sub.s64 %rd18, %rd5, %rd4;
-; SM70-NEXT: selp.b64 %rd19, %rd18, %rd5, %p1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd5, %rd19;
-; SM70-NEXT: setp.ne.b64 %p2, %rd34, %rd5;
+; SM70-NEXT: mov.b64 %rd7, %rd34;
+; SM70-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
+; SM70-NEXT: sub.s64 %rd18, %rd7, %rd5;
+; SM70-NEXT: selp.b64 %rd19, %rd18, %rd7, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
; SM70-NEXT: @%p2 bra $L__BB223_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
-; SM70-NEXT: $L__BB223_3: // %atomicrmw.start12
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd6, %rd35;
-; SM70-NEXT: setp.ge.u64 %p3, %rd6, %rd3;
-; SM70-NEXT: sub.s64 %rd20, %rd6, %rd3;
-; SM70-NEXT: selp.b64 %rd21, %rd20, %rd6, %p3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd6, %rd21;
-; SM70-NEXT: setp.ne.b64 %p4, %rd35, %rd6;
+; SM70-NEXT: $L__BB223_3: // %atomicrmw.start10
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd35;
+; SM70-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
+; SM70-NEXT: sub.s64 %rd20, %rd8, %rd6;
+; SM70-NEXT: selp.b64 %rd21, %rd20, %rd8, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
; SM70-NEXT: @%p4 bra $L__BB223_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end11
+; SM70-NEXT: // %bb.4: // %atomicrmw.end9
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
-; SM70-NEXT: $L__BB223_5: // %atomicrmw.start24
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd8, %rd36;
-; SM70-NEXT: setp.ge.u64 %p5, %rd8, %rd2;
-; SM70-NEXT: sub.s64 %rd22, %rd8, %rd2;
-; SM70-NEXT: selp.b64 %rd23, %rd22, %rd8, %p5;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd8, %rd23;
-; SM70-NEXT: setp.ne.b64 %p6, %rd36, %rd8;
+; SM70-NEXT: $L__BB223_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd36;
+; SM70-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
+; SM70-NEXT: sub.s64 %rd22, %rd9, %rd3;
+; SM70-NEXT: selp.b64 %rd23, %rd22, %rd9, %p5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
; SM70-NEXT: @%p6 bra $L__BB223_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end23
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
-; SM70-NEXT: $L__BB223_7: // %atomicrmw.start32
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd9, %rd37;
-; SM70-NEXT: setp.ge.u64 %p7, %rd9, %rd7;
-; SM70-NEXT: sub.s64 %rd24, %rd9, %rd7;
-; SM70-NEXT: selp.b64 %rd25, %rd24, %rd9, %p7;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd9, %rd25;
-; SM70-NEXT: setp.ne.b64 %p8, %rd37, %rd9;
+; SM70-NEXT: $L__BB223_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd37;
+; SM70-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
+; SM70-NEXT: sub.s64 %rd24, %rd10, %rd4;
+; SM70-NEXT: selp.b64 %rd25, %rd24, %rd10, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
; SM70-NEXT: @%p8 bra $L__BB223_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end31
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
-; SM70-NEXT: $L__BB223_9: // %atomicrmw.start48
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd12, %rd38;
-; SM70-NEXT: setp.ge.u64 %p9, %rd12, %rd1;
-; SM70-NEXT: sub.s64 %rd26, %rd12, %rd1;
-; SM70-NEXT: selp.b64 %rd27, %rd26, %rd12, %p9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd12, %rd27;
-; SM70-NEXT: setp.ne.b64 %p10, %rd38, %rd12;
+; SM70-NEXT: $L__BB223_9: // %atomicrmw.start40
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd38;
+; SM70-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
+; SM70-NEXT: sub.s64 %rd26, %rd13, %rd1;
+; SM70-NEXT: selp.b64 %rd27, %rd26, %rd13, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM70-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
; SM70-NEXT: @%p10 bra $L__BB223_9;
-; SM70-NEXT: // %bb.10: // %atomicrmw.end47
+; SM70-NEXT: // %bb.10: // %atomicrmw.end39
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
-; SM70-NEXT: $L__BB223_11: // %atomicrmw.start56
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd13, %rd39;
-; SM70-NEXT: setp.ge.u64 %p11, %rd13, %rd11;
-; SM70-NEXT: sub.s64 %rd28, %rd13, %rd11;
-; SM70-NEXT: selp.b64 %rd29, %rd28, %rd13, %p11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd13, %rd29;
-; SM70-NEXT: setp.ne.b64 %p12, %rd39, %rd13;
+; SM70-NEXT: $L__BB223_11: // %atomicrmw.start46
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd39;
+; SM70-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
+; SM70-NEXT: sub.s64 %rd28, %rd14, %rd2;
+; SM70-NEXT: selp.b64 %rd29, %rd28, %rd14, %p11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM70-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
; SM70-NEXT: @%p12 bra $L__BB223_11;
-; SM70-NEXT: // %bb.12: // %atomicrmw.end55
+; SM70-NEXT: // %bb.12: // %atomicrmw.end45
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
-; SM70-NEXT: $L__BB223_13: // %atomicrmw.start68
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd15, %rd40;
-; SM70-NEXT: setp.ge.u64 %p13, %rd15, %rd10;
-; SM70-NEXT: sub.s64 %rd30, %rd15, %rd10;
-; SM70-NEXT: selp.b64 %rd31, %rd30, %rd15, %p13;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd40, [%rd17+48], %rd15, %rd31;
-; SM70-NEXT: setp.ne.b64 %p14, %rd40, %rd15;
+; SM70-NEXT: $L__BB223_13: // %atomicrmw.start56
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
+; SM70-NEXT: sub.s64 %rd30, %rd40, %rd11;
+; SM70-NEXT: selp.b64 %rd31, %rd30, %rd40, %p13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM70-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
+; SM70-NEXT: mov.b64 %rd40, %rd15;
; SM70-NEXT: @%p14 bra $L__BB223_13;
-; SM70-NEXT: // %bb.14: // %atomicrmw.end67
+; SM70-NEXT: // %bb.14: // %atomicrmw.end55
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
-; SM70-NEXT: $L__BB223_15: // %atomicrmw.start76
+; SM70-NEXT: $L__BB223_15: // %atomicrmw.start62
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.ge.u64 %p15, %rd41, %rd14;
-; SM70-NEXT: sub.s64 %rd32, %rd41, %rd14;
+; SM70-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
+; SM70-NEXT: sub.s64 %rd32, %rd41, %rd12;
; SM70-NEXT: selp.b64 %rd33, %rd32, %rd41, %p15;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM70-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
; SM70-NEXT: mov.b64 %rd41, %rd16;
; SM70-NEXT: @%p16 bra $L__BB223_15;
-; SM70-NEXT: // %bb.16: // %atomicrmw.end75
+; SM70-NEXT: // %bb.16: // %atomicrmw.end61
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd40, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
@@ -9185,9 +9159,9 @@ define <1 x i16> @usub_sat_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NEXT: and.b32 %r5, %r4, 3;
@@ -9432,21 +9406,21 @@ define <1 x i32> @usub_sat_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b32 %r1, [usub_sat_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT: $L__BB232_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: max.u32 %r3, %r5, %r1;
-; SM70-NEXT: sub.s32 %r4, %r3, %r1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: max.u32 %r3, %r5, %r2;
+; SM70-NEXT: sub.s32 %r4, %r3, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
; SM70-NEXT: @%p1 bra $L__BB232_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -9676,21 +9650,21 @@ define <1 x i64> @usub_sat_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-NEXT: .reg .b64 %rd<7>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd2];
; SM70-NEXT: $L__BB236_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: max.u64 %rd4, %rd6, %rd1;
-; SM70-NEXT: sub.s64 %rd5, %rd4, %rd1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
-; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
-; SM70-NEXT: mov.b64 %rd6, %rd2;
+; SM70-NEXT: max.u64 %rd4, %rd6, %rd3;
+; SM70-NEXT: sub.s64 %rd5, %rd4, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5;
+; SM70-NEXT: setp.ne.b64 %p1, %rd1, %rd6;
+; SM70-NEXT: mov.b64 %rd6, %rd1;
; SM70-NEXT: @%p1 bra $L__BB236_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -9705,29 +9679,29 @@ define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd5, [usub_sat_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
; SM70-NEXT: $L__BB237_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd3, %rd10;
-; SM70-NEXT: max.u64 %rd6, %rd3, %rd2;
-; SM70-NEXT: sub.s64 %rd7, %rd6, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd10, [%rd5], %rd3, %rd7;
-; SM70-NEXT: setp.ne.b64 %p1, %rd10, %rd3;
+; SM70-NEXT: max.u64 %rd6, %rd10, %rd1;
+; SM70-NEXT: sub.s64 %rd7, %rd6, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
+; SM70-NEXT: mov.b64 %rd10, %rd3;
; SM70-NEXT: @%p1 bra $L__BB237_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
-; SM70-NEXT: $L__BB237_3: // %atomicrmw.start4
+; SM70-NEXT: $L__BB237_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: max.u64 %rd8, %rd11, %rd1;
-; SM70-NEXT: sub.s64 %rd9, %rd8, %rd1;
+; SM70-NEXT: max.u64 %rd8, %rd11, %rd2;
+; SM70-NEXT: sub.s64 %rd9, %rd8, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
; SM70-NEXT: mov.b64 %rd11, %rd4;
; SM70-NEXT: @%p2 bra $L__BB237_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end3
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd4};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -9742,50 +9716,50 @@ define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd9, [usub_sat_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
; SM70-NEXT: $L__BB238_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd4, %rd18;
-; SM70-NEXT: max.u64 %rd10, %rd4, %rd3;
+; SM70-NEXT: mov.b64 %rd5, %rd18;
+; SM70-NEXT: max.u64 %rd10, %rd5, %rd3;
; SM70-NEXT: sub.s64 %rd11, %rd10, %rd3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd4, %rd11;
-; SM70-NEXT: setp.ne.b64 %p1, %rd18, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
; SM70-NEXT: @%p1 bra $L__BB238_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
-; SM70-NEXT: $L__BB238_3: // %atomicrmw.start8
+; SM70-NEXT: $L__BB238_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd19;
-; SM70-NEXT: max.u64 %rd12, %rd5, %rd2;
-; SM70-NEXT: sub.s64 %rd13, %rd12, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd5, %rd13;
-; SM70-NEXT: setp.ne.b64 %p2, %rd19, %rd5;
+; SM70-NEXT: mov.b64 %rd6, %rd19;
+; SM70-NEXT: max.u64 %rd12, %rd6, %rd4;
+; SM70-NEXT: sub.s64 %rd13, %rd12, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM70-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
; SM70-NEXT: @%p2 bra $L__BB238_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end7
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
-; SM70-NEXT: $L__BB238_5: // %atomicrmw.start20
+; SM70-NEXT: $L__BB238_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd7, %rd20;
-; SM70-NEXT: max.u64 %rd14, %rd7, %rd1;
+; SM70-NEXT: max.u64 %rd14, %rd20, %rd1;
; SM70-NEXT: sub.s64 %rd15, %rd14, %rd1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd20, [%rd9+16], %rd7, %rd15;
-; SM70-NEXT: setp.ne.b64 %p3, %rd20, %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
+; SM70-NEXT: mov.b64 %rd20, %rd7;
; SM70-NEXT: @%p3 bra $L__BB238_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
-; SM70-NEXT: $L__BB238_7: // %atomicrmw.start28
+; SM70-NEXT: $L__BB238_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: max.u64 %rd16, %rd21, %rd6;
-; SM70-NEXT: sub.s64 %rd17, %rd16, %rd6;
+; SM70-NEXT: max.u64 %rd16, %rd21, %rd2;
+; SM70-NEXT: sub.s64 %rd17, %rd16, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
; SM70-NEXT: mov.b64 %rd21, %rd8;
; SM70-NEXT: @%p4 bra $L__BB238_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end27
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd20, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
@@ -9801,92 +9775,92 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd17, [usub_sat_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [usub_sat_acq_rel_v8i64_global_cta_param_1+48];
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [usub_sat_acq_rel_v8i64_global_cta_param_1+32];
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [usub_sat_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
; SM70-NEXT: $L__BB239_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd34;
-; SM70-NEXT: max.u64 %rd18, %rd5, %rd4;
-; SM70-NEXT: sub.s64 %rd19, %rd18, %rd4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd5, %rd19;
-; SM70-NEXT: setp.ne.b64 %p1, %rd34, %rd5;
+; SM70-NEXT: mov.b64 %rd7, %rd34;
+; SM70-NEXT: max.u64 %rd18, %rd7, %rd5;
+; SM70-NEXT: sub.s64 %rd19, %rd18, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM70-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
; SM70-NEXT: @%p1 bra $L__BB239_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
-; SM70-NEXT: $L__BB239_3: // %atomicrmw.start12
+; SM70-NEXT: $L__BB239_3: // %atomicrmw.start10
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd6, %rd35;
-; SM70-NEXT: max.u64 %rd20, %rd6, %rd3;
-; SM70-NEXT: sub.s64 %rd21, %rd20, %rd3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd6, %rd21;
-; SM70-NEXT: setp.ne.b64 %p2, %rd35, %rd6;
+; SM70-NEXT: mov.b64 %rd8, %rd35;
+; SM70-NEXT: max.u64 %rd20, %rd8, %rd6;
+; SM70-NEXT: sub.s64 %rd21, %rd20, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM70-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
; SM70-NEXT: @%p2 bra $L__BB239_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end11
+; SM70-NEXT: // %bb.4: // %atomicrmw.end9
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
-; SM70-NEXT: $L__BB239_5: // %atomicrmw.start24
+; SM70-NEXT: $L__BB239_5: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd8, %rd36;
-; SM70-NEXT: max.u64 %rd22, %rd8, %rd2;
-; SM70-NEXT: sub.s64 %rd23, %rd22, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd8, %rd23;
-; SM70-NEXT: setp.ne.b64 %p3, %rd36, %rd8;
+; SM70-NEXT: mov.b64 %rd9, %rd36;
+; SM70-NEXT: max.u64 %rd22, %rd9, %rd3;
+; SM70-NEXT: sub.s64 %rd23, %rd22, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM70-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
; SM70-NEXT: @%p3 bra $L__BB239_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end23
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
-; SM70-NEXT: $L__BB239_7: // %atomicrmw.start32
+; SM70-NEXT: $L__BB239_7: // %atomicrmw.start26
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd9, %rd37;
-; SM70-NEXT: max.u64 %rd24, %rd9, %rd7;
-; SM70-NEXT: sub.s64 %rd25, %rd24, %rd7;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd9, %rd25;
-; SM70-NEXT: setp.ne.b64 %p4, %rd37, %rd9;
+; SM70-NEXT: mov.b64 %rd10, %rd37;
+; SM70-NEXT: max.u64 %rd24, %rd10, %rd4;
+; SM70-NEXT: sub.s64 %rd25, %rd24, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM70-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
; SM70-NEXT: @%p4 bra $L__BB239_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end31
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
-; SM70-NEXT: $L__BB239_9: // %atomicrmw.start48
+; SM70-NEXT: $L__BB239_9: // %atomicrmw.start40
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd12, %rd38;
-; SM70-NEXT: max.u64 %rd26, %rd12, %rd1;
+; SM70-NEXT: mov.b64 %rd13, %rd38;
+; SM70-NEXT: max.u64 %rd26, %rd13, %rd1;
; SM70-NEXT: sub.s64 %rd27, %rd26, %rd1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd12, %rd27;
-; SM70-NEXT: setp.ne.b64 %p5, %rd38, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM70-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
; SM70-NEXT: @%p5 bra $L__BB239_9;
-; SM70-NEXT: // %bb.10: // %atomicrmw.end47
+; SM70-NEXT: // %bb.10: // %atomicrmw.end39
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
-; SM70-NEXT: $L__BB239_11: // %atomicrmw.start56
+; SM70-NEXT: $L__BB239_11: // %atomicrmw.start46
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd13, %rd39;
-; SM70-NEXT: max.u64 %rd28, %rd13, %rd11;
-; SM70-NEXT: sub.s64 %rd29, %rd28, %rd11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd13, %rd29;
-; SM70-NEXT: setp.ne.b64 %p6, %rd39, %rd13;
+; SM70-NEXT: mov.b64 %rd14, %rd39;
+; SM70-NEXT: max.u64 %rd28, %rd14, %rd2;
+; SM70-NEXT: sub.s64 %rd29, %rd28, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM70-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
; SM70-NEXT: @%p6 bra $L__BB239_11;
-; SM70-NEXT: // %bb.12: // %atomicrmw.end55
+; SM70-NEXT: // %bb.12: // %atomicrmw.end45
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
-; SM70-NEXT: $L__BB239_13: // %atomicrmw.start68
+; SM70-NEXT: $L__BB239_13: // %atomicrmw.start56
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd15, %rd40;
-; SM70-NEXT: max.u64 %rd30, %rd15, %rd10;
-; SM70-NEXT: sub.s64 %rd31, %rd30, %rd10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd40, [%rd17+48], %rd15, %rd31;
-; SM70-NEXT: setp.ne.b64 %p7, %rd40, %rd15;
+; SM70-NEXT: max.u64 %rd30, %rd40, %rd11;
+; SM70-NEXT: sub.s64 %rd31, %rd30, %rd11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
+; SM70-NEXT: mov.b64 %rd40, %rd15;
; SM70-NEXT: @%p7 bra $L__BB239_13;
-; SM70-NEXT: // %bb.14: // %atomicrmw.end67
+; SM70-NEXT: // %bb.14: // %atomicrmw.end55
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
-; SM70-NEXT: $L__BB239_15: // %atomicrmw.start76
+; SM70-NEXT: $L__BB239_15: // %atomicrmw.start62
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: max.u64 %rd32, %rd41, %rd14;
-; SM70-NEXT: sub.s64 %rd33, %rd32, %rd14;
+; SM70-NEXT: max.u64 %rd32, %rd41, %rd12;
+; SM70-NEXT: sub.s64 %rd33, %rd32, %rd12;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
; SM70-NEXT: mov.b64 %rd41, %rd16;
; SM70-NEXT: @%p8 bra $L__BB239_15;
-; SM70-NEXT: // %bb.16: // %atomicrmw.end75
+; SM70-NEXT: // %bb.16: // %atomicrmw.end61
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd40, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
@@ -9903,20 +9877,20 @@ define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB240_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB240_1;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_v1f32_global_cta(
@@ -9926,10 +9900,8 @@ define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
@@ -9940,10 +9912,8 @@ define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-FTZ-DISALLOW-NEXT: ret;
;
@@ -9954,10 +9924,8 @@ define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
@@ -10340,20 +10308,20 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB244_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r3, %r4, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB244_1;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v1f32_global_cta(
@@ -10363,11 +10331,9 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r2, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r2;
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r3, [%rd1], %r2;
; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r3;
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
@@ -10378,11 +10344,9 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r2, %r1;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r2;
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r3, [%rd1], %r2;
; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r3;
; SM70-FTZ-DISALLOW-NEXT: ret;
;
@@ -10393,11 +10357,9 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r2, %r1;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r2;
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r3, [%rd1], %r2;
; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r3;
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
@@ -10447,14 +10409,14 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
-; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r3, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1], %r3;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r5, %r2;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r5;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r3, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r4, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r4;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r3;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM70-NOFTZ-ALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v2f32_global_cta(
@@ -10464,14 +10426,14 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r3, %r1;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1], %r3;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r5, %r2;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r5;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r3, %r2;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r4, %r1;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r4;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r3;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v2f32_global_cta(
@@ -10481,14 +10443,14 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r3, %r1;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1], %r3;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r5, %r2;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r5;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r3, %r2;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r4, %r1;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r4;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r3;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM70-FTZ-ALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
@@ -10498,57 +10460,61 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
; SM70-NOFTZ-DISALLOW: {
; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<17>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v4f32_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f32 %r2, %r6;
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f32 %r1, %r5;
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f32 %r4, %r8;
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f32 %r3, %r7;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r13, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB246_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r5, %r9, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r6, %r10, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r9, %r13, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r10, %r14, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r10;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r9;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r14;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r13, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB246_1;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r15, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB246_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r7, %r11, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r7;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r8, %r12, %r4;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r11, %r15, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r12, %r16, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r12;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r16;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r15, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB246_3;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
@@ -10558,18 +10524,18 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
-; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r5, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1], %r5;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r5, %r4;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r6, %r3;
; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r7, %r2;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+4], %r7;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r9, %r3;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+8], %r9;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r11, %r4;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r11;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r8, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r8;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r7;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r6;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r5;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM70-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
@@ -10579,18 +10545,18 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r5, %r1;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1], %r5;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r5, %r4;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r6, %r3;
; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r7, %r2;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+4], %r7;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r9, %r3;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+8], %r9;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r11, %r4;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r11;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r8, %r1;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r8;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r7;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r6;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r5;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
@@ -10600,18 +10566,18 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r5, %r1;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1], %r5;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r5, %r4;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r6, %r3;
; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r7, %r2;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+4], %r7;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r9, %r3;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+8], %r9;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r11, %r4;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r11;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r8, %r1;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r8;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r7;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r6;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r5;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM70-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
@@ -10621,101 +10587,109 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
; SM70-NOFTZ-DISALLOW: {
; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<5>;
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<25>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<33>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<42>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r9, %r10, %r11, %r12}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f32 %r4, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f32 %r3, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f32 %r2, %r10;
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f32 %r1, %r9;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r13, %r14, %r15, %r16}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f32 %r6, %r14;
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f32 %r5, %r13;
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f32 %r8, %r16;
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f32 %r7, %r15;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r25, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB247_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r9, %r17, %r5;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r10, %r18, %r6;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r17, %r25, %r5;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r18, %r26, %r6;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r18;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r17;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r26;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r25, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB247_1;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r27, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB247_3: // %atomicrmw.start6
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r11, %r19, %r7;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r11;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r12, %r20, %r8;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r19, %r27, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r19;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r20, %r28, %r8;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r20;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r19;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r28;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r27, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB247_3;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r29, %rd22;
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB247_5: // %atomicrmw.start16
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r13, %r21, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd23, %r13;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r14, %r22, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r21, %r29, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd23, %r21;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r22, %r30, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd24, %r22;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd25, %rd24, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd26, %rd23, %rd25;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd27, %r21;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd27, %r29;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd28, %r30;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd29, %rd28, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd30, %rd27, %rd29;
; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r29, %rd31;
; SM70-NOFTZ-DISALLOW-NEXT: @%p3 bra $L__BB247_5;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r31, %rd32;
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB247_7: // %atomicrmw.start22
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r15, %r23, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd33, %r15;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r16, %r24, %r4;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r23, %r31, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd33, %r23;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r24, %r32, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd34, %r24;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd35, %rd34, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd36, %rd33, %rd35;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd37, %r23;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd37, %r31;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd38, %r32;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd39, %rd38, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd40, %rd37, %rd39;
; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r31, %rd41;
; SM70-NOFTZ-DISALLOW-NEXT: @%p4 bra $L__BB247_7;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
@@ -10725,28 +10699,28 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
-; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r9, %r5;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1], %r9;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r11, %r6;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+4], %r11;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r13, %r7;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+8], %r13;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r15, %r8;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+12], %r15;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r17, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r18, [%rd1+16], %r17;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r19, %r2;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r20, [%rd1+20], %r19;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r21, %r3;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r22, [%rd1+24], %r21;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r23, %r4;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r23;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r5, %r4;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r6, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r7, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r8, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r9, %r10, %r11, %r12}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r13, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r14, %r11;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r15, %r10;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f32 %r16, %r9;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
-; SM70-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r17, [%rd1], %r16;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r18, [%rd1+4], %r15;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r19, [%rd1+8], %r14;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r20, [%rd1+12], %r13;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r21, [%rd1+16], %r8;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r22, [%rd1+20], %r7;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r23, [%rd1+24], %r6;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r5;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
@@ -10756,28 +10730,28 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
-; SM70-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r9, %r5;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1], %r9;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r11, %r6;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+4], %r11;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r13, %r7;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+8], %r13;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r15, %r8;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+12], %r15;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r17, %r1;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r18, [%rd1+16], %r17;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r19, %r2;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r20, [%rd1+20], %r19;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r21, %r3;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r22, [%rd1+24], %r21;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r23, %r4;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r23;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r5, %r4;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r6, %r3;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r7, %r2;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r8, %r1;
+; SM70-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r9, %r10, %r11, %r12}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r13, %r12;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r14, %r11;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r15, %r10;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r16, %r9;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r17, [%rd1], %r16;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r18, [%rd1+4], %r15;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r19, [%rd1+8], %r14;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r20, [%rd1+12], %r13;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r21, [%rd1+16], %r8;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r22, [%rd1+20], %r7;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r23, [%rd1+24], %r6;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r5;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
-; SM70-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM70-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
@@ -10787,28 +10761,28 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
-; SM70-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r9, %r5;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1], %r9;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r11, %r6;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+4], %r11;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r13, %r7;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+8], %r13;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r15, %r8;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+12], %r15;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r17, %r1;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r18, [%rd1+16], %r17;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r19, %r2;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r20, [%rd1+20], %r19;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r21, %r3;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r22, [%rd1+24], %r21;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r23, %r4;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r23;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r5, %r4;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r6, %r3;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r7, %r2;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r8, %r1;
+; SM70-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r9, %r10, %r11, %r12}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r13, %r12;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r14, %r11;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r15, %r10;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f32 %r16, %r9;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
-; SM70-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r17, [%rd1], %r16;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r18, [%rd1+4], %r15;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r19, [%rd1+8], %r14;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r20, [%rd1+12], %r13;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r21, [%rd1+16], %r8;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r22, [%rd1+20], %r7;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r23, [%rd1+24], %r6;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r5;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
@@ -10822,20 +10796,20 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB248_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: min.f32 %r3, %r4, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: min.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB248_1;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v1f32_global_cta(
@@ -10845,20 +10819,20 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT: $L__BB248_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-ALLOW-NEXT: min.f32 %r3, %r4, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: min.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM70-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB248_1;
; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_v1f32_global_cta(
@@ -10868,20 +10842,20 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-FTZ-DISALLOW-NEXT: $L__BB248_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-DISALLOW-NEXT: min.ftz.f32 %r3, %r4, %r1;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM70-FTZ-DISALLOW-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM70-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB248_1;
; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_v1f32_global_cta(
@@ -10891,20 +10865,20 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-FTZ-ALLOW-NEXT: $L__BB248_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-ALLOW-NEXT: min.ftz.f32 %r3, %r4, %r1;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM70-FTZ-ALLOW-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM70-FTZ-ALLOW-NEXT: @%p1 bra $L__BB248_1;
; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -11694,20 +11668,20 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB252_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: max.f32 %r3, %r4, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: max.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB252_1;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v1f32_global_cta(
@@ -11717,20 +11691,20 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT: $L__BB252_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-ALLOW-NEXT: max.f32 %r3, %r4, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: max.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM70-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB252_1;
; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_v1f32_global_cta(
@@ -11740,20 +11714,20 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-FTZ-DISALLOW-NEXT: $L__BB252_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-DISALLOW-NEXT: max.ftz.f32 %r3, %r4, %r1;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM70-FTZ-DISALLOW-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM70-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB252_1;
; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_v1f32_global_cta(
@@ -11763,20 +11737,20 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-FTZ-ALLOW-NEXT: $L__BB252_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-ALLOW-NEXT: max.ftz.f32 %r3, %r4, %r1;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM70-FTZ-ALLOW-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM70-FTZ-ALLOW-NEXT: @%p1 bra $L__BB252_1;
; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -12566,26 +12540,26 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB256_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r7, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: min.f32 %r4, %r7, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: min.f32 %r4, %r7, %r2;
; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p3, %r7, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: @%p4 bra $L__BB256_1;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v1f32_global_cta(
@@ -12595,26 +12569,26 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT: $L__BB256_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
-; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r7, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: min.f32 %r4, %r7, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: min.f32 %r4, %r7, %r2;
; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM70-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p3, %r7, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r7, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
; SM70-NOFTZ-ALLOW-NEXT: @%p4 bra $L__BB256_1;
; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v1f32_global_cta(
@@ -12624,26 +12598,26 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-FTZ-DISALLOW-NEXT: $L__BB256_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
-; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r1;
-; SM70-FTZ-DISALLOW-NEXT: min.ftz.f32 %r4, %r7, %r1;
+; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM70-FTZ-DISALLOW-NEXT: min.ftz.f32 %r4, %r7, %r2;
; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM70-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r1;
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r7, %r2;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
; SM70-FTZ-DISALLOW-NEXT: @%p4 bra $L__BB256_1;
; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_v1f32_global_cta(
@@ -12653,26 +12627,26 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-FTZ-ALLOW-NEXT: $L__BB256_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
-; SM70-FTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r1;
-; SM70-FTZ-ALLOW-NEXT: min.ftz.f32 %r4, %r7, %r1;
+; SM70-FTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM70-FTZ-ALLOW-NEXT: min.ftz.f32 %r4, %r7, %r2;
; SM70-FTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM70-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r1;
+; SM70-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
; SM70-FTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r7, %r2;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
; SM70-FTZ-ALLOW-NEXT: @%p4 bra $L__BB256_1;
; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -13798,26 +13772,26 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB260_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r7, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: max.f32 %r4, %r7, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: max.f32 %r4, %r7, %r2;
; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p3, %r7, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: @%p4 bra $L__BB260_1;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v1f32_global_cta(
@@ -13827,26 +13801,26 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT: $L__BB260_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
-; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r7, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: max.f32 %r4, %r7, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: max.f32 %r4, %r7, %r2;
; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM70-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p3, %r7, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r7, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
; SM70-NOFTZ-ALLOW-NEXT: @%p4 bra $L__BB260_1;
; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v1f32_global_cta(
@@ -13856,26 +13830,26 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-FTZ-DISALLOW-NEXT: $L__BB260_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
-; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r1;
-; SM70-FTZ-DISALLOW-NEXT: max.ftz.f32 %r4, %r7, %r1;
+; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM70-FTZ-DISALLOW-NEXT: max.ftz.f32 %r4, %r7, %r2;
; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM70-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r1;
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r7, %r2;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
; SM70-FTZ-DISALLOW-NEXT: @%p4 bra $L__BB260_1;
; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v1f32_global_cta(
@@ -13885,26 +13859,26 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-FTZ-ALLOW-NEXT: $L__BB260_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
-; SM70-FTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r1, %p1;
-; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r1;
-; SM70-FTZ-ALLOW-NEXT: max.ftz.f32 %r4, %r7, %r1;
+; SM70-FTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM70-FTZ-ALLOW-NEXT: max.ftz.f32 %r4, %r7, %r2;
; SM70-FTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM70-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r1;
+; SM70-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
; SM70-FTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
-; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r2, %r7;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r7, %r2;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
; SM70-FTZ-ALLOW-NEXT: @%p4 bra $L__BB260_1;
; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -15029,10 +15003,8 @@ define <1 x double> @fadd_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f64_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd3, [%rd1], %rd2;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.add.f64 %rd3, [%rd1], %rd2;
; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
@@ -15116,11 +15088,9 @@ define <1 x double> @fsub_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1f64_global_cta_param_1];
; SM70-NEXT: neg.f64 %rd3, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd4, [%rd1], %rd3;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.add.f64 %rd4, [%rd1], %rd3;
; SM70-NEXT: st.param.b64 [func_retval0], %rd4;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
@@ -15134,14 +15104,14 @@ define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fsub_acq_rel_v2f64_global_cta_param_1];
-; SM70-NEXT: neg.f64 %rd4, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd5, [%rd1], %rd4;
-; SM70-NEXT: neg.f64 %rd6, %rd3;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd7, [%rd1+8], %rd6;
+; SM70-NEXT: neg.f64 %rd4, %rd3;
+; SM70-NEXT: neg.f64 %rd5, %rd2;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd7};
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd6, [%rd1], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd7, [%rd1+8], %rd4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -15154,20 +15124,20 @@ define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fsub_acq_rel_v4f64_global_cta_param_1];
-; SM70-NEXT: neg.f64 %rd6, %rd4;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd7, [%rd1], %rd6;
-; SM70-NEXT: neg.f64 %rd8, %rd5;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd9, [%rd1+8], %rd8;
-; SM70-NEXT: neg.f64 %rd10, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd11, [%rd1+16], %rd10;
-; SM70-NEXT: neg.f64 %rd12, %rd3;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd13, [%rd1+24], %rd12;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd11, %rd13};
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd7, %rd9};
+; SM70-NEXT: neg.f64 %rd4, %rd3;
+; SM70-NEXT: neg.f64 %rd5, %rd2;
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [fsub_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: neg.f64 %rd8, %rd7;
+; SM70-NEXT: neg.f64 %rd9, %rd6;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd10, [%rd1], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd11, [%rd1+8], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd12, [%rd1+16], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd13, [%rd1+24], %rd4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
ret <4 x double> %retval
@@ -15180,32 +15150,32 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f64_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fsub_acq_rel_v8f64_global_cta_param_1+48];
-; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
-; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [fsub_acq_rel_v8f64_global_cta_param_1];
-; SM70-NEXT: neg.f64 %rd10, %rd8;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd11, [%rd1], %rd10;
-; SM70-NEXT: neg.f64 %rd12, %rd9;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd13, [%rd1+8], %rd12;
-; SM70-NEXT: neg.f64 %rd14, %rd6;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd15, [%rd1+16], %rd14;
-; SM70-NEXT: neg.f64 %rd16, %rd7;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd17, [%rd1+24], %rd16;
-; SM70-NEXT: neg.f64 %rd18, %rd4;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd19, [%rd1+32], %rd18;
-; SM70-NEXT: neg.f64 %rd20, %rd5;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd21, [%rd1+40], %rd20;
-; SM70-NEXT: neg.f64 %rd22, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd23, [%rd1+48], %rd22;
-; SM70-NEXT: neg.f64 %rd24, %rd3;
-; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd25, [%rd1+56], %rd24;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd23, %rd25};
-; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd19, %rd21};
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd15, %rd17};
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd11, %rd13};
+; SM70-NEXT: neg.f64 %rd4, %rd3;
+; SM70-NEXT: neg.f64 %rd5, %rd2;
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: neg.f64 %rd8, %rd7;
+; SM70-NEXT: neg.f64 %rd9, %rd6;
+; SM70-NEXT: ld.param.v2.b64 {%rd10, %rd11}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: neg.f64 %rd12, %rd11;
+; SM70-NEXT: neg.f64 %rd13, %rd10;
+; SM70-NEXT: ld.param.v2.b64 {%rd14, %rd15}, [fsub_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: neg.f64 %rd16, %rd15;
+; SM70-NEXT: neg.f64 %rd17, %rd14;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd18, [%rd1], %rd17;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd19, [%rd1+8], %rd16;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd20, [%rd1+16], %rd13;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd21, [%rd1+24], %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd22, [%rd1+32], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd23, [%rd1+40], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd24, [%rd1+48], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd25, [%rd1+56], %rd4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd24, %rd25};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd22, %rd23};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd20, %rd21};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -15218,20 +15188,20 @@ define <1 x double> @fmin_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-NEXT: .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd2];
; SM70-NEXT: $L__BB272_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: min.f64 %rd4, %rd5, %rd1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
-; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
-; SM70-NEXT: mov.b64 %rd5, %rd2;
+; SM70-NEXT: min.f64 %rd4, %rd5, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd5, %rd4;
+; SM70-NEXT: setp.ne.b64 %p1, %rd1, %rd5;
+; SM70-NEXT: mov.b64 %rd5, %rd1;
; SM70-NEXT: @%p1 bra $L__BB272_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -15246,27 +15216,27 @@ define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd5, [fmin_acq_rel_v2f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [fmin_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd5];
; SM70-NEXT: $L__BB273_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd3, %rd8;
-; SM70-NEXT: min.f64 %rd6, %rd3, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd5], %rd3, %rd6;
-; SM70-NEXT: setp.ne.b64 %p1, %rd8, %rd3;
+; SM70-NEXT: min.f64 %rd6, %rd8, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM70-NEXT: mov.b64 %rd8, %rd3;
; SM70-NEXT: @%p1 bra $L__BB273_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd5+8];
-; SM70-NEXT: $L__BB273_3: // %atomicrmw.start4
+; SM70-NEXT: $L__BB273_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: min.f64 %rd7, %rd9, %rd1;
+; SM70-NEXT: min.f64 %rd7, %rd9, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
; SM70-NEXT: mov.b64 %rd9, %rd4;
; SM70-NEXT: @%p2 bra $L__BB273_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end3
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd8, %rd4};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -15281,46 +15251,46 @@ define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd9, [fmin_acq_rel_v4f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd9];
; SM70-NEXT: $L__BB274_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd4, %rd14;
-; SM70-NEXT: min.f64 %rd10, %rd4, %rd3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd4, %rd10;
-; SM70-NEXT: setp.ne.b64 %p1, %rd14, %rd4;
+; SM70-NEXT: mov.b64 %rd5, %rd14;
+; SM70-NEXT: min.f64 %rd10, %rd5, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
; SM70-NEXT: @%p1 bra $L__BB274_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd9+8];
-; SM70-NEXT: $L__BB274_3: // %atomicrmw.start8
+; SM70-NEXT: $L__BB274_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd15;
-; SM70-NEXT: min.f64 %rd11, %rd5, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd5, %rd11;
-; SM70-NEXT: setp.ne.b64 %p2, %rd15, %rd5;
+; SM70-NEXT: mov.b64 %rd6, %rd15;
+; SM70-NEXT: min.f64 %rd11, %rd6, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
; SM70-NEXT: @%p2 bra $L__BB274_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end7
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd9+16];
-; SM70-NEXT: $L__BB274_5: // %atomicrmw.start19
+; SM70-NEXT: $L__BB274_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd7, %rd16;
-; SM70-NEXT: min.f64 %rd12, %rd7, %rd1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd9+16], %rd7, %rd12;
-; SM70-NEXT: setp.ne.b64 %p3, %rd16, %rd7;
+; SM70-NEXT: min.f64 %rd12, %rd16, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM70-NEXT: mov.b64 %rd16, %rd7;
; SM70-NEXT: @%p3 bra $L__BB274_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end18
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd9+24];
-; SM70-NEXT: $L__BB274_7: // %atomicrmw.start26
+; SM70-NEXT: $L__BB274_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: min.f64 %rd13, %rd17, %rd6;
+; SM70-NEXT: min.f64 %rd13, %rd17, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
; SM70-NEXT: mov.b64 %rd17, %rd8;
; SM70-NEXT: @%p4 bra $L__BB274_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd16, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
@@ -15336,84 +15306,84 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd17, [fmin_acq_rel_v8f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [fmin_acq_rel_v8f64_global_cta_param_1+48];
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [fmin_acq_rel_v8f64_global_cta_param_1+32];
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [fmin_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmin_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmin_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd17];
; SM70-NEXT: $L__BB275_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd26;
-; SM70-NEXT: min.f64 %rd18, %rd5, %rd4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd5, %rd18;
-; SM70-NEXT: setp.ne.b64 %p1, %rd26, %rd5;
+; SM70-NEXT: mov.b64 %rd7, %rd26;
+; SM70-NEXT: min.f64 %rd18, %rd7, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM70-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
; SM70-NEXT: @%p1 bra $L__BB275_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd17+8];
-; SM70-NEXT: $L__BB275_3: // %atomicrmw.start12
+; SM70-NEXT: $L__BB275_3: // %atomicrmw.start10
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd6, %rd27;
-; SM70-NEXT: min.f64 %rd19, %rd6, %rd3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd6, %rd19;
-; SM70-NEXT: setp.ne.b64 %p2, %rd27, %rd6;
+; SM70-NEXT: mov.b64 %rd8, %rd27;
+; SM70-NEXT: min.f64 %rd19, %rd8, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
; SM70-NEXT: @%p2 bra $L__BB275_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end11
+; SM70-NEXT: // %bb.4: // %atomicrmw.end9
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd17+16];
-; SM70-NEXT: $L__BB275_5: // %atomicrmw.start23
+; SM70-NEXT: $L__BB275_5: // %atomicrmw.start19
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd8, %rd28;
-; SM70-NEXT: min.f64 %rd20, %rd8, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd8, %rd20;
-; SM70-NEXT: setp.ne.b64 %p3, %rd28, %rd8;
+; SM70-NEXT: mov.b64 %rd9, %rd28;
+; SM70-NEXT: min.f64 %rd20, %rd9, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM70-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
; SM70-NEXT: @%p3 bra $L__BB275_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end22
+; SM70-NEXT: // %bb.6: // %atomicrmw.end18
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd17+24];
-; SM70-NEXT: $L__BB275_7: // %atomicrmw.start30
+; SM70-NEXT: $L__BB275_7: // %atomicrmw.start24
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd9, %rd29;
-; SM70-NEXT: min.f64 %rd21, %rd9, %rd7;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd9, %rd21;
-; SM70-NEXT: setp.ne.b64 %p4, %rd29, %rd9;
+; SM70-NEXT: mov.b64 %rd10, %rd29;
+; SM70-NEXT: min.f64 %rd21, %rd10, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
; SM70-NEXT: @%p4 bra $L__BB275_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end29
+; SM70-NEXT: // %bb.8: // %atomicrmw.end23
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd17+32];
-; SM70-NEXT: $L__BB275_9: // %atomicrmw.start45
+; SM70-NEXT: $L__BB275_9: // %atomicrmw.start37
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd12, %rd30;
-; SM70-NEXT: min.f64 %rd22, %rd12, %rd1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd12, %rd22;
-; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd12;
+; SM70-NEXT: mov.b64 %rd13, %rd30;
+; SM70-NEXT: min.f64 %rd22, %rd13, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
; SM70-NEXT: @%p5 bra $L__BB275_9;
-; SM70-NEXT: // %bb.10: // %atomicrmw.end44
+; SM70-NEXT: // %bb.10: // %atomicrmw.end36
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd17+40];
-; SM70-NEXT: $L__BB275_11: // %atomicrmw.start52
+; SM70-NEXT: $L__BB275_11: // %atomicrmw.start42
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd13, %rd31;
-; SM70-NEXT: min.f64 %rd23, %rd13, %rd11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd13, %rd23;
-; SM70-NEXT: setp.ne.b64 %p6, %rd31, %rd13;
+; SM70-NEXT: mov.b64 %rd14, %rd31;
+; SM70-NEXT: min.f64 %rd23, %rd14, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
; SM70-NEXT: @%p6 bra $L__BB275_11;
-; SM70-NEXT: // %bb.12: // %atomicrmw.end51
+; SM70-NEXT: // %bb.12: // %atomicrmw.end41
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd17+48];
-; SM70-NEXT: $L__BB275_13: // %atomicrmw.start63
+; SM70-NEXT: $L__BB275_13: // %atomicrmw.start51
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd15, %rd32;
-; SM70-NEXT: min.f64 %rd24, %rd15, %rd10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd32, [%rd17+48], %rd15, %rd24;
-; SM70-NEXT: setp.ne.b64 %p7, %rd32, %rd15;
+; SM70-NEXT: min.f64 %rd24, %rd32, %rd11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM70-NEXT: mov.b64 %rd32, %rd15;
; SM70-NEXT: @%p7 bra $L__BB275_13;
-; SM70-NEXT: // %bb.14: // %atomicrmw.end62
+; SM70-NEXT: // %bb.14: // %atomicrmw.end50
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd17+56];
-; SM70-NEXT: $L__BB275_15: // %atomicrmw.start70
+; SM70-NEXT: $L__BB275_15: // %atomicrmw.start56
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: min.f64 %rd25, %rd33, %rd14;
+; SM70-NEXT: min.f64 %rd25, %rd33, %rd12;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
; SM70-NEXT: mov.b64 %rd33, %rd16;
; SM70-NEXT: @%p8 bra $L__BB275_15;
-; SM70-NEXT: // %bb.16: // %atomicrmw.end69
+; SM70-NEXT: // %bb.16: // %atomicrmw.end55
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd32, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
@@ -15429,20 +15399,20 @@ define <1 x double> @fmax_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-NEXT: .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd2];
; SM70-NEXT: $L__BB276_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: max.f64 %rd4, %rd5, %rd1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
-; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
-; SM70-NEXT: mov.b64 %rd5, %rd2;
+; SM70-NEXT: max.f64 %rd4, %rd5, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd5, %rd4;
+; SM70-NEXT: setp.ne.b64 %p1, %rd1, %rd5;
+; SM70-NEXT: mov.b64 %rd5, %rd1;
; SM70-NEXT: @%p1 bra $L__BB276_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -15457,27 +15427,27 @@ define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd5, [fmax_acq_rel_v2f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [fmax_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd5];
; SM70-NEXT: $L__BB277_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd3, %rd8;
-; SM70-NEXT: max.f64 %rd6, %rd3, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd5], %rd3, %rd6;
-; SM70-NEXT: setp.ne.b64 %p1, %rd8, %rd3;
+; SM70-NEXT: max.f64 %rd6, %rd8, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM70-NEXT: mov.b64 %rd8, %rd3;
; SM70-NEXT: @%p1 bra $L__BB277_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd5+8];
-; SM70-NEXT: $L__BB277_3: // %atomicrmw.start4
+; SM70-NEXT: $L__BB277_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: max.f64 %rd7, %rd9, %rd1;
+; SM70-NEXT: max.f64 %rd7, %rd9, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
; SM70-NEXT: mov.b64 %rd9, %rd4;
; SM70-NEXT: @%p2 bra $L__BB277_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end3
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd8, %rd4};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -15492,46 +15462,46 @@ define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd9, [fmax_acq_rel_v4f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd9];
; SM70-NEXT: $L__BB278_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd4, %rd14;
-; SM70-NEXT: max.f64 %rd10, %rd4, %rd3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd4, %rd10;
-; SM70-NEXT: setp.ne.b64 %p1, %rd14, %rd4;
+; SM70-NEXT: mov.b64 %rd5, %rd14;
+; SM70-NEXT: max.f64 %rd10, %rd5, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
; SM70-NEXT: @%p1 bra $L__BB278_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd9+8];
-; SM70-NEXT: $L__BB278_3: // %atomicrmw.start8
+; SM70-NEXT: $L__BB278_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd15;
-; SM70-NEXT: max.f64 %rd11, %rd5, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd5, %rd11;
-; SM70-NEXT: setp.ne.b64 %p2, %rd15, %rd5;
+; SM70-NEXT: mov.b64 %rd6, %rd15;
+; SM70-NEXT: max.f64 %rd11, %rd6, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
; SM70-NEXT: @%p2 bra $L__BB278_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end7
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd9+16];
-; SM70-NEXT: $L__BB278_5: // %atomicrmw.start19
+; SM70-NEXT: $L__BB278_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd7, %rd16;
-; SM70-NEXT: max.f64 %rd12, %rd7, %rd1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd9+16], %rd7, %rd12;
-; SM70-NEXT: setp.ne.b64 %p3, %rd16, %rd7;
+; SM70-NEXT: max.f64 %rd12, %rd16, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM70-NEXT: mov.b64 %rd16, %rd7;
; SM70-NEXT: @%p3 bra $L__BB278_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end18
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd9+24];
-; SM70-NEXT: $L__BB278_7: // %atomicrmw.start26
+; SM70-NEXT: $L__BB278_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: max.f64 %rd13, %rd17, %rd6;
+; SM70-NEXT: max.f64 %rd13, %rd17, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
; SM70-NEXT: mov.b64 %rd17, %rd8;
; SM70-NEXT: @%p4 bra $L__BB278_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd16, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
@@ -15547,84 +15517,84 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd17, [fmax_acq_rel_v8f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [fmax_acq_rel_v8f64_global_cta_param_1+48];
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [fmax_acq_rel_v8f64_global_cta_param_1+32];
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [fmax_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmax_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmax_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd17];
; SM70-NEXT: $L__BB279_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd26;
-; SM70-NEXT: max.f64 %rd18, %rd5, %rd4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd5, %rd18;
-; SM70-NEXT: setp.ne.b64 %p1, %rd26, %rd5;
+; SM70-NEXT: mov.b64 %rd7, %rd26;
+; SM70-NEXT: max.f64 %rd18, %rd7, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM70-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
; SM70-NEXT: @%p1 bra $L__BB279_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd17+8];
-; SM70-NEXT: $L__BB279_3: // %atomicrmw.start12
+; SM70-NEXT: $L__BB279_3: // %atomicrmw.start10
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd6, %rd27;
-; SM70-NEXT: max.f64 %rd19, %rd6, %rd3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd6, %rd19;
-; SM70-NEXT: setp.ne.b64 %p2, %rd27, %rd6;
+; SM70-NEXT: mov.b64 %rd8, %rd27;
+; SM70-NEXT: max.f64 %rd19, %rd8, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
; SM70-NEXT: @%p2 bra $L__BB279_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end11
+; SM70-NEXT: // %bb.4: // %atomicrmw.end9
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd17+16];
-; SM70-NEXT: $L__BB279_5: // %atomicrmw.start23
+; SM70-NEXT: $L__BB279_5: // %atomicrmw.start19
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd8, %rd28;
-; SM70-NEXT: max.f64 %rd20, %rd8, %rd2;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd8, %rd20;
-; SM70-NEXT: setp.ne.b64 %p3, %rd28, %rd8;
+; SM70-NEXT: mov.b64 %rd9, %rd28;
+; SM70-NEXT: max.f64 %rd20, %rd9, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM70-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
; SM70-NEXT: @%p3 bra $L__BB279_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end22
+; SM70-NEXT: // %bb.6: // %atomicrmw.end18
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd17+24];
-; SM70-NEXT: $L__BB279_7: // %atomicrmw.start30
+; SM70-NEXT: $L__BB279_7: // %atomicrmw.start24
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd9, %rd29;
-; SM70-NEXT: max.f64 %rd21, %rd9, %rd7;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd9, %rd21;
-; SM70-NEXT: setp.ne.b64 %p4, %rd29, %rd9;
+; SM70-NEXT: mov.b64 %rd10, %rd29;
+; SM70-NEXT: max.f64 %rd21, %rd10, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
; SM70-NEXT: @%p4 bra $L__BB279_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end29
+; SM70-NEXT: // %bb.8: // %atomicrmw.end23
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd17+32];
-; SM70-NEXT: $L__BB279_9: // %atomicrmw.start45
+; SM70-NEXT: $L__BB279_9: // %atomicrmw.start37
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd12, %rd30;
-; SM70-NEXT: max.f64 %rd22, %rd12, %rd1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd12, %rd22;
-; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd12;
+; SM70-NEXT: mov.b64 %rd13, %rd30;
+; SM70-NEXT: max.f64 %rd22, %rd13, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
; SM70-NEXT: @%p5 bra $L__BB279_9;
-; SM70-NEXT: // %bb.10: // %atomicrmw.end44
+; SM70-NEXT: // %bb.10: // %atomicrmw.end36
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd17+40];
-; SM70-NEXT: $L__BB279_11: // %atomicrmw.start52
+; SM70-NEXT: $L__BB279_11: // %atomicrmw.start42
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd13, %rd31;
-; SM70-NEXT: max.f64 %rd23, %rd13, %rd11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd13, %rd23;
-; SM70-NEXT: setp.ne.b64 %p6, %rd31, %rd13;
+; SM70-NEXT: mov.b64 %rd14, %rd31;
+; SM70-NEXT: max.f64 %rd23, %rd14, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
; SM70-NEXT: @%p6 bra $L__BB279_11;
-; SM70-NEXT: // %bb.12: // %atomicrmw.end51
+; SM70-NEXT: // %bb.12: // %atomicrmw.end41
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd17+48];
-; SM70-NEXT: $L__BB279_13: // %atomicrmw.start63
+; SM70-NEXT: $L__BB279_13: // %atomicrmw.start51
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd15, %rd32;
-; SM70-NEXT: max.f64 %rd24, %rd15, %rd10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd32, [%rd17+48], %rd15, %rd24;
-; SM70-NEXT: setp.ne.b64 %p7, %rd32, %rd15;
+; SM70-NEXT: max.f64 %rd24, %rd32, %rd11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM70-NEXT: mov.b64 %rd32, %rd15;
; SM70-NEXT: @%p7 bra $L__BB279_13;
-; SM70-NEXT: // %bb.14: // %atomicrmw.end62
+; SM70-NEXT: // %bb.14: // %atomicrmw.end50
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd17+56];
-; SM70-NEXT: $L__BB279_15: // %atomicrmw.start70
+; SM70-NEXT: $L__BB279_15: // %atomicrmw.start56
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: max.f64 %rd25, %rd33, %rd14;
+; SM70-NEXT: max.f64 %rd25, %rd33, %rd12;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
; SM70-NEXT: mov.b64 %rd33, %rd16;
; SM70-NEXT: @%p8 bra $L__BB279_15;
-; SM70-NEXT: // %bb.16: // %atomicrmw.end69
+; SM70-NEXT: // %bb.16: // %atomicrmw.end55
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd32, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
@@ -15640,26 +15610,26 @@ define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: .reg .b64 %rd<9>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd3];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd2];
; SM70-NEXT: $L__BB280_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.eq.b64 %p1, %rd8, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd4, %rd8, %rd1, %p1;
-; SM70-NEXT: setp.nan.f64 %p2, %rd8, %rd1;
-; SM70-NEXT: min.f64 %rd5, %rd8, %rd1;
+; SM70-NEXT: selp.f64 %rd4, %rd8, %rd3, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd8, %rd3;
+; SM70-NEXT: min.f64 %rd5, %rd8, %rd3;
; SM70-NEXT: selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
-; SM70-NEXT: setp.eq.f64 %p3, %rd8, %rd1;
+; SM70-NEXT: setp.eq.f64 %p3, %rd8, %rd3;
; SM70-NEXT: selp.f64 %rd7, %rd4, %rd6, %p3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
-; SM70-NEXT: setp.ne.b64 %p4, %rd2, %rd8;
-; SM70-NEXT: mov.b64 %rd8, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd8, %rd7;
+; SM70-NEXT: setp.ne.b64 %p4, %rd1, %rd8;
+; SM70-NEXT: mov.b64 %rd8, %rd1;
; SM70-NEXT: @%p4 bra $L__BB280_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -15674,39 +15644,39 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd5, [fminimum_acq_rel_v2f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [fminimum_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd5];
; SM70-NEXT: $L__BB281_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd3, %rd14;
-; SM70-NEXT: setp.eq.b64 %p1, %rd3, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd6, %rd3, %rd2, %p1;
-; SM70-NEXT: setp.nan.f64 %p2, %rd3, %rd2;
-; SM70-NEXT: min.f64 %rd7, %rd3, %rd2;
+; SM70-NEXT: setp.eq.b64 %p1, %rd14, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd6, %rd14, %rd1, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd14, %rd1;
+; SM70-NEXT: min.f64 %rd7, %rd14, %rd1;
; SM70-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p2;
-; SM70-NEXT: setp.eq.f64 %p3, %rd3, %rd2;
+; SM70-NEXT: setp.eq.f64 %p3, %rd14, %rd1;
; SM70-NEXT: selp.f64 %rd9, %rd6, %rd8, %p3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd5], %rd3, %rd9;
-; SM70-NEXT: setp.ne.b64 %p4, %rd14, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd14, %rd9;
+; SM70-NEXT: setp.ne.b64 %p4, %rd3, %rd14;
+; SM70-NEXT: mov.b64 %rd14, %rd3;
; SM70-NEXT: @%p4 bra $L__BB281_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd5+8];
-; SM70-NEXT: $L__BB281_3: // %atomicrmw.start4
+; SM70-NEXT: $L__BB281_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.eq.b64 %p5, %rd15, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd10, %rd15, %rd1, %p5;
-; SM70-NEXT: setp.nan.f64 %p6, %rd15, %rd1;
-; SM70-NEXT: min.f64 %rd11, %rd15, %rd1;
+; SM70-NEXT: selp.f64 %rd10, %rd15, %rd2, %p5;
+; SM70-NEXT: setp.nan.f64 %p6, %rd15, %rd2;
+; SM70-NEXT: min.f64 %rd11, %rd15, %rd2;
; SM70-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
-; SM70-NEXT: setp.eq.f64 %p7, %rd15, %rd1;
+; SM70-NEXT: setp.eq.f64 %p7, %rd15, %rd2;
; SM70-NEXT: selp.f64 %rd13, %rd10, %rd12, %p7;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd15, %rd13;
; SM70-NEXT: setp.ne.b64 %p8, %rd4, %rd15;
; SM70-NEXT: mov.b64 %rd15, %rd4;
; SM70-NEXT: @%p8 bra $L__BB281_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end3
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd4};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -15721,70 +15691,70 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd9];
; SM70-NEXT: $L__BB282_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd4, %rd26;
-; SM70-NEXT: setp.eq.b64 %p1, %rd4, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd10, %rd4, %rd3, %p1;
-; SM70-NEXT: setp.nan.f64 %p2, %rd4, %rd3;
-; SM70-NEXT: min.f64 %rd11, %rd4, %rd3;
+; SM70-NEXT: mov.b64 %rd5, %rd26;
+; SM70-NEXT: setp.eq.b64 %p1, %rd5, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd10, %rd5, %rd3, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd5, %rd3;
+; SM70-NEXT: min.f64 %rd11, %rd5, %rd3;
; SM70-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p2;
-; SM70-NEXT: setp.eq.f64 %p3, %rd4, %rd3;
+; SM70-NEXT: setp.eq.f64 %p3, %rd5, %rd3;
; SM70-NEXT: selp.f64 %rd13, %rd10, %rd12, %p3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd9], %rd4, %rd13;
-; SM70-NEXT: setp.ne.b64 %p4, %rd26, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd9], %rd5, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd26, %rd5;
; SM70-NEXT: @%p4 bra $L__BB282_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd9+8];
-; SM70-NEXT: $L__BB282_3: // %atomicrmw.start8
+; SM70-NEXT: $L__BB282_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd27;
-; SM70-NEXT: setp.eq.b64 %p5, %rd5, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd14, %rd5, %rd2, %p5;
-; SM70-NEXT: setp.nan.f64 %p6, %rd5, %rd2;
-; SM70-NEXT: min.f64 %rd15, %rd5, %rd2;
+; SM70-NEXT: mov.b64 %rd6, %rd27;
+; SM70-NEXT: setp.eq.b64 %p5, %rd6, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd14, %rd6, %rd4, %p5;
+; SM70-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
+; SM70-NEXT: min.f64 %rd15, %rd6, %rd4;
; SM70-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
-; SM70-NEXT: setp.eq.f64 %p7, %rd5, %rd2;
+; SM70-NEXT: setp.eq.f64 %p7, %rd6, %rd4;
; SM70-NEXT: selp.f64 %rd17, %rd14, %rd16, %p7;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd9+8], %rd5, %rd17;
-; SM70-NEXT: setp.ne.b64 %p8, %rd27, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd9+8], %rd6, %rd17;
+; SM70-NEXT: setp.ne.b64 %p8, %rd27, %rd6;
; SM70-NEXT: @%p8 bra $L__BB282_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end7
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd9+16];
-; SM70-NEXT: $L__BB282_5: // %atomicrmw.start19
+; SM70-NEXT: $L__BB282_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd7, %rd28;
-; SM70-NEXT: setp.eq.b64 %p9, %rd7, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd18, %rd7, %rd1, %p9;
-; SM70-NEXT: setp.nan.f64 %p10, %rd7, %rd1;
-; SM70-NEXT: min.f64 %rd19, %rd7, %rd1;
+; SM70-NEXT: setp.eq.b64 %p9, %rd28, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd18, %rd28, %rd1, %p9;
+; SM70-NEXT: setp.nan.f64 %p10, %rd28, %rd1;
+; SM70-NEXT: min.f64 %rd19, %rd28, %rd1;
; SM70-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p10;
-; SM70-NEXT: setp.eq.f64 %p11, %rd7, %rd1;
+; SM70-NEXT: setp.eq.f64 %p11, %rd28, %rd1;
; SM70-NEXT: selp.f64 %rd21, %rd18, %rd20, %p11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd9+16], %rd7, %rd21;
-; SM70-NEXT: setp.ne.b64 %p12, %rd28, %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd28, %rd21;
+; SM70-NEXT: setp.ne.b64 %p12, %rd7, %rd28;
+; SM70-NEXT: mov.b64 %rd28, %rd7;
; SM70-NEXT: @%p12 bra $L__BB282_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end18
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd9+24];
-; SM70-NEXT: $L__BB282_7: // %atomicrmw.start26
+; SM70-NEXT: $L__BB282_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.eq.b64 %p13, %rd29, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd22, %rd29, %rd6, %p13;
-; SM70-NEXT: setp.nan.f64 %p14, %rd29, %rd6;
-; SM70-NEXT: min.f64 %rd23, %rd29, %rd6;
+; SM70-NEXT: selp.f64 %rd22, %rd29, %rd2, %p13;
+; SM70-NEXT: setp.nan.f64 %p14, %rd29, %rd2;
+; SM70-NEXT: min.f64 %rd23, %rd29, %rd2;
; SM70-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p14;
-; SM70-NEXT: setp.eq.f64 %p15, %rd29, %rd6;
+; SM70-NEXT: setp.eq.f64 %p15, %rd29, %rd2;
; SM70-NEXT: selp.f64 %rd25, %rd22, %rd24, %p15;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd29, %rd25;
; SM70-NEXT: setp.ne.b64 %p16, %rd8, %rd29;
; SM70-NEXT: mov.b64 %rd29, %rd8;
; SM70-NEXT: @%p16 bra $L__BB282_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
@@ -15800,132 +15770,132 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [fminimum_acq_rel_v8f64_global_cta_param_1+48];
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [fminimum_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fminimum_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd50, [%rd17];
; SM70-NEXT: $L__BB283_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd50;
-; SM70-NEXT: setp.eq.b64 %p1, %rd5, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd18, %rd5, %rd4, %p1;
-; SM70-NEXT: setp.nan.f64 %p2, %rd5, %rd4;
-; SM70-NEXT: min.f64 %rd19, %rd5, %rd4;
+; SM70-NEXT: mov.b64 %rd7, %rd50;
+; SM70-NEXT: setp.eq.b64 %p1, %rd7, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd18, %rd7, %rd5, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd7, %rd5;
+; SM70-NEXT: min.f64 %rd19, %rd7, %rd5;
; SM70-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p2;
-; SM70-NEXT: setp.eq.f64 %p3, %rd5, %rd4;
+; SM70-NEXT: setp.eq.f64 %p3, %rd7, %rd5;
; SM70-NEXT: selp.f64 %rd21, %rd18, %rd20, %p3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd50, [%rd17], %rd5, %rd21;
-; SM70-NEXT: setp.ne.b64 %p4, %rd50, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd50, [%rd17], %rd7, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd50, %rd7;
; SM70-NEXT: @%p4 bra $L__BB283_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd51, [%rd17+8];
-; SM70-NEXT: $L__BB283_3: // %atomicrmw.start12
+; SM70-NEXT: $L__BB283_3: // %atomicrmw.start10
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd6, %rd51;
-; SM70-NEXT: setp.eq.b64 %p5, %rd6, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd22, %rd6, %rd3, %p5;
-; SM70-NEXT: setp.nan.f64 %p6, %rd6, %rd3;
-; SM70-NEXT: min.f64 %rd23, %rd6, %rd3;
+; SM70-NEXT: mov.b64 %rd8, %rd51;
+; SM70-NEXT: setp.eq.b64 %p5, %rd8, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd22, %rd8, %rd6, %p5;
+; SM70-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
+; SM70-NEXT: min.f64 %rd23, %rd8, %rd6;
; SM70-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
-; SM70-NEXT: setp.eq.f64 %p7, %rd6, %rd3;
+; SM70-NEXT: setp.eq.f64 %p7, %rd8, %rd6;
; SM70-NEXT: selp.f64 %rd25, %rd22, %rd24, %p7;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd51, [%rd17+8], %rd6, %rd25;
-; SM70-NEXT: setp.ne.b64 %p8, %rd51, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd51, [%rd17+8], %rd8, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd51, %rd8;
; SM70-NEXT: @%p8 bra $L__BB283_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end11
+; SM70-NEXT: // %bb.4: // %atomicrmw.end9
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd52, [%rd17+16];
-; SM70-NEXT: $L__BB283_5: // %atomicrmw.start23
+; SM70-NEXT: $L__BB283_5: // %atomicrmw.start19
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd8, %rd52;
-; SM70-NEXT: setp.eq.b64 %p9, %rd8, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd26, %rd8, %rd2, %p9;
-; SM70-NEXT: setp.nan.f64 %p10, %rd8, %rd2;
-; SM70-NEXT: min.f64 %rd27, %rd8, %rd2;
+; SM70-NEXT: mov.b64 %rd9, %rd52;
+; SM70-NEXT: setp.eq.b64 %p9, %rd9, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd26, %rd9, %rd3, %p9;
+; SM70-NEXT: setp.nan.f64 %p10, %rd9, %rd3;
+; SM70-NEXT: min.f64 %rd27, %rd9, %rd3;
; SM70-NEXT: selp.f64 %rd28, 0d7FF8000000000000, %rd27, %p10;
-; SM70-NEXT: setp.eq.f64 %p11, %rd8, %rd2;
+; SM70-NEXT: setp.eq.f64 %p11, %rd9, %rd3;
; SM70-NEXT: selp.f64 %rd29, %rd26, %rd28, %p11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd52, [%rd17+16], %rd8, %rd29;
-; SM70-NEXT: setp.ne.b64 %p12, %rd52, %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd52, [%rd17+16], %rd9, %rd29;
+; SM70-NEXT: setp.ne.b64 %p12, %rd52, %rd9;
; SM70-NEXT: @%p12 bra $L__BB283_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end22
+; SM70-NEXT: // %bb.6: // %atomicrmw.end18
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd53, [%rd17+24];
-; SM70-NEXT: $L__BB283_7: // %atomicrmw.start30
+; SM70-NEXT: $L__BB283_7: // %atomicrmw.start24
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd9, %rd53;
-; SM70-NEXT: setp.eq.b64 %p13, %rd9, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd30, %rd9, %rd7, %p13;
-; SM70-NEXT: setp.nan.f64 %p14, %rd9, %rd7;
-; SM70-NEXT: min.f64 %rd31, %rd9, %rd7;
+; SM70-NEXT: mov.b64 %rd10, %rd53;
+; SM70-NEXT: setp.eq.b64 %p13, %rd10, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd30, %rd10, %rd4, %p13;
+; SM70-NEXT: setp.nan.f64 %p14, %rd10, %rd4;
+; SM70-NEXT: min.f64 %rd31, %rd10, %rd4;
; SM70-NEXT: selp.f64 %rd32, 0d7FF8000000000000, %rd31, %p14;
-; SM70-NEXT: setp.eq.f64 %p15, %rd9, %rd7;
+; SM70-NEXT: setp.eq.f64 %p15, %rd10, %rd4;
; SM70-NEXT: selp.f64 %rd33, %rd30, %rd32, %p15;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd53, [%rd17+24], %rd9, %rd33;
-; SM70-NEXT: setp.ne.b64 %p16, %rd53, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd53, [%rd17+24], %rd10, %rd33;
+; SM70-NEXT: setp.ne.b64 %p16, %rd53, %rd10;
; SM70-NEXT: @%p16 bra $L__BB283_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end29
+; SM70-NEXT: // %bb.8: // %atomicrmw.end23
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd54, [%rd17+32];
-; SM70-NEXT: $L__BB283_9: // %atomicrmw.start45
+; SM70-NEXT: $L__BB283_9: // %atomicrmw.start37
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd12, %rd54;
-; SM70-NEXT: setp.eq.b64 %p17, %rd12, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd34, %rd12, %rd1, %p17;
-; SM70-NEXT: setp.nan.f64 %p18, %rd12, %rd1;
-; SM70-NEXT: min.f64 %rd35, %rd12, %rd1;
+; SM70-NEXT: mov.b64 %rd13, %rd54;
+; SM70-NEXT: setp.eq.b64 %p17, %rd13, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd34, %rd13, %rd1, %p17;
+; SM70-NEXT: setp.nan.f64 %p18, %rd13, %rd1;
+; SM70-NEXT: min.f64 %rd35, %rd13, %rd1;
; SM70-NEXT: selp.f64 %rd36, 0d7FF8000000000000, %rd35, %p18;
-; SM70-NEXT: setp.eq.f64 %p19, %rd12, %rd1;
+; SM70-NEXT: setp.eq.f64 %p19, %rd13, %rd1;
; SM70-NEXT: selp.f64 %rd37, %rd34, %rd36, %p19;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd54, [%rd17+32], %rd12, %rd37;
-; SM70-NEXT: setp.ne.b64 %p20, %rd54, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd54, [%rd17+32], %rd13, %rd37;
+; SM70-NEXT: setp.ne.b64 %p20, %rd54, %rd13;
; SM70-NEXT: @%p20 bra $L__BB283_9;
-; SM70-NEXT: // %bb.10: // %atomicrmw.end44
+; SM70-NEXT: // %bb.10: // %atomicrmw.end36
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd55, [%rd17+40];
-; SM70-NEXT: $L__BB283_11: // %atomicrmw.start52
+; SM70-NEXT: $L__BB283_11: // %atomicrmw.start42
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd13, %rd55;
-; SM70-NEXT: setp.eq.b64 %p21, %rd13, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd38, %rd13, %rd11, %p21;
-; SM70-NEXT: setp.nan.f64 %p22, %rd13, %rd11;
-; SM70-NEXT: min.f64 %rd39, %rd13, %rd11;
+; SM70-NEXT: mov.b64 %rd14, %rd55;
+; SM70-NEXT: setp.eq.b64 %p21, %rd14, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd38, %rd14, %rd2, %p21;
+; SM70-NEXT: setp.nan.f64 %p22, %rd14, %rd2;
+; SM70-NEXT: min.f64 %rd39, %rd14, %rd2;
; SM70-NEXT: selp.f64 %rd40, 0d7FF8000000000000, %rd39, %p22;
-; SM70-NEXT: setp.eq.f64 %p23, %rd13, %rd11;
+; SM70-NEXT: setp.eq.f64 %p23, %rd14, %rd2;
; SM70-NEXT: selp.f64 %rd41, %rd38, %rd40, %p23;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd55, [%rd17+40], %rd13, %rd41;
-; SM70-NEXT: setp.ne.b64 %p24, %rd55, %rd13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd55, [%rd17+40], %rd14, %rd41;
+; SM70-NEXT: setp.ne.b64 %p24, %rd55, %rd14;
; SM70-NEXT: @%p24 bra $L__BB283_11;
-; SM70-NEXT: // %bb.12: // %atomicrmw.end51
+; SM70-NEXT: // %bb.12: // %atomicrmw.end41
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd56, [%rd17+48];
-; SM70-NEXT: $L__BB283_13: // %atomicrmw.start63
+; SM70-NEXT: $L__BB283_13: // %atomicrmw.start51
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd15, %rd56;
-; SM70-NEXT: setp.eq.b64 %p25, %rd15, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd42, %rd15, %rd10, %p25;
-; SM70-NEXT: setp.nan.f64 %p26, %rd15, %rd10;
-; SM70-NEXT: min.f64 %rd43, %rd15, %rd10;
+; SM70-NEXT: setp.eq.b64 %p25, %rd56, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd42, %rd56, %rd11, %p25;
+; SM70-NEXT: setp.nan.f64 %p26, %rd56, %rd11;
+; SM70-NEXT: min.f64 %rd43, %rd56, %rd11;
; SM70-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
-; SM70-NEXT: setp.eq.f64 %p27, %rd15, %rd10;
+; SM70-NEXT: setp.eq.f64 %p27, %rd56, %rd11;
; SM70-NEXT: selp.f64 %rd45, %rd42, %rd44, %p27;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd56, [%rd17+48], %rd15, %rd45;
-; SM70-NEXT: setp.ne.b64 %p28, %rd56, %rd15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd56, %rd45;
+; SM70-NEXT: setp.ne.b64 %p28, %rd15, %rd56;
+; SM70-NEXT: mov.b64 %rd56, %rd15;
; SM70-NEXT: @%p28 bra $L__BB283_13;
-; SM70-NEXT: // %bb.14: // %atomicrmw.end62
+; SM70-NEXT: // %bb.14: // %atomicrmw.end50
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd57, [%rd17+56];
-; SM70-NEXT: $L__BB283_15: // %atomicrmw.start70
+; SM70-NEXT: $L__BB283_15: // %atomicrmw.start56
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.eq.b64 %p29, %rd57, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd46, %rd57, %rd14, %p29;
-; SM70-NEXT: setp.nan.f64 %p30, %rd57, %rd14;
-; SM70-NEXT: min.f64 %rd47, %rd57, %rd14;
+; SM70-NEXT: selp.f64 %rd46, %rd57, %rd12, %p29;
+; SM70-NEXT: setp.nan.f64 %p30, %rd57, %rd12;
+; SM70-NEXT: min.f64 %rd47, %rd57, %rd12;
; SM70-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p30;
-; SM70-NEXT: setp.eq.f64 %p31, %rd57, %rd14;
+; SM70-NEXT: setp.eq.f64 %p31, %rd57, %rd12;
; SM70-NEXT: selp.f64 %rd49, %rd46, %rd48, %p31;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd57, %rd49;
; SM70-NEXT: setp.ne.b64 %p32, %rd16, %rd57;
; SM70-NEXT: mov.b64 %rd57, %rd16;
; SM70-NEXT: @%p32 bra $L__BB283_15;
-; SM70-NEXT: // %bb.16: // %atomicrmw.end69
+; SM70-NEXT: // %bb.16: // %atomicrmw.end55
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd56, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd54, %rd55};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
@@ -15941,26 +15911,26 @@ define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: .reg .b64 %rd<9>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd3];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd2];
; SM70-NEXT: $L__BB284_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.eq.b64 %p1, %rd8, 0;
-; SM70-NEXT: selp.f64 %rd4, %rd8, %rd1, %p1;
-; SM70-NEXT: setp.nan.f64 %p2, %rd8, %rd1;
-; SM70-NEXT: max.f64 %rd5, %rd8, %rd1;
+; SM70-NEXT: selp.f64 %rd4, %rd8, %rd3, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd8, %rd3;
+; SM70-NEXT: max.f64 %rd5, %rd8, %rd3;
; SM70-NEXT: selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
-; SM70-NEXT: setp.eq.f64 %p3, %rd8, %rd1;
+; SM70-NEXT: setp.eq.f64 %p3, %rd8, %rd3;
; SM70-NEXT: selp.f64 %rd7, %rd4, %rd6, %p3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
-; SM70-NEXT: setp.ne.b64 %p4, %rd2, %rd8;
-; SM70-NEXT: mov.b64 %rd8, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd8, %rd7;
+; SM70-NEXT: setp.ne.b64 %p4, %rd1, %rd8;
+; SM70-NEXT: mov.b64 %rd8, %rd1;
; SM70-NEXT: @%p4 bra $L__BB284_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -15975,39 +15945,39 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd5, [fmaximum_acq_rel_v2f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd1}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd5];
; SM70-NEXT: $L__BB285_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd3, %rd14;
-; SM70-NEXT: setp.eq.b64 %p1, %rd3, 0;
-; SM70-NEXT: selp.f64 %rd6, %rd3, %rd2, %p1;
-; SM70-NEXT: setp.nan.f64 %p2, %rd3, %rd2;
-; SM70-NEXT: max.f64 %rd7, %rd3, %rd2;
+; SM70-NEXT: setp.eq.b64 %p1, %rd14, 0;
+; SM70-NEXT: selp.f64 %rd6, %rd14, %rd1, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd14, %rd1;
+; SM70-NEXT: max.f64 %rd7, %rd14, %rd1;
; SM70-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p2;
-; SM70-NEXT: setp.eq.f64 %p3, %rd3, %rd2;
+; SM70-NEXT: setp.eq.f64 %p3, %rd14, %rd1;
; SM70-NEXT: selp.f64 %rd9, %rd6, %rd8, %p3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd5], %rd3, %rd9;
-; SM70-NEXT: setp.ne.b64 %p4, %rd14, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd14, %rd9;
+; SM70-NEXT: setp.ne.b64 %p4, %rd3, %rd14;
+; SM70-NEXT: mov.b64 %rd14, %rd3;
; SM70-NEXT: @%p4 bra $L__BB285_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd5+8];
-; SM70-NEXT: $L__BB285_3: // %atomicrmw.start4
+; SM70-NEXT: $L__BB285_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.eq.b64 %p5, %rd15, 0;
-; SM70-NEXT: selp.f64 %rd10, %rd15, %rd1, %p5;
-; SM70-NEXT: setp.nan.f64 %p6, %rd15, %rd1;
-; SM70-NEXT: max.f64 %rd11, %rd15, %rd1;
+; SM70-NEXT: selp.f64 %rd10, %rd15, %rd2, %p5;
+; SM70-NEXT: setp.nan.f64 %p6, %rd15, %rd2;
+; SM70-NEXT: max.f64 %rd11, %rd15, %rd2;
; SM70-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
-; SM70-NEXT: setp.eq.f64 %p7, %rd15, %rd1;
+; SM70-NEXT: setp.eq.f64 %p7, %rd15, %rd2;
; SM70-NEXT: selp.f64 %rd13, %rd10, %rd12, %p7;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd15, %rd13;
; SM70-NEXT: setp.ne.b64 %p8, %rd4, %rd15;
; SM70-NEXT: mov.b64 %rd15, %rd4;
; SM70-NEXT: @%p8 bra $L__BB285_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end3
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd4};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -16022,70 +15992,70 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd6}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd9];
; SM70-NEXT: $L__BB286_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd4, %rd26;
-; SM70-NEXT: setp.eq.b64 %p1, %rd4, 0;
-; SM70-NEXT: selp.f64 %rd10, %rd4, %rd3, %p1;
-; SM70-NEXT: setp.nan.f64 %p2, %rd4, %rd3;
-; SM70-NEXT: max.f64 %rd11, %rd4, %rd3;
+; SM70-NEXT: mov.b64 %rd5, %rd26;
+; SM70-NEXT: setp.eq.b64 %p1, %rd5, 0;
+; SM70-NEXT: selp.f64 %rd10, %rd5, %rd3, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd5, %rd3;
+; SM70-NEXT: max.f64 %rd11, %rd5, %rd3;
; SM70-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p2;
-; SM70-NEXT: setp.eq.f64 %p3, %rd4, %rd3;
+; SM70-NEXT: setp.eq.f64 %p3, %rd5, %rd3;
; SM70-NEXT: selp.f64 %rd13, %rd10, %rd12, %p3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd9], %rd4, %rd13;
-; SM70-NEXT: setp.ne.b64 %p4, %rd26, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd9], %rd5, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd26, %rd5;
; SM70-NEXT: @%p4 bra $L__BB286_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd9+8];
-; SM70-NEXT: $L__BB286_3: // %atomicrmw.start8
+; SM70-NEXT: $L__BB286_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd27;
-; SM70-NEXT: setp.eq.b64 %p5, %rd5, 0;
-; SM70-NEXT: selp.f64 %rd14, %rd5, %rd2, %p5;
-; SM70-NEXT: setp.nan.f64 %p6, %rd5, %rd2;
-; SM70-NEXT: max.f64 %rd15, %rd5, %rd2;
+; SM70-NEXT: mov.b64 %rd6, %rd27;
+; SM70-NEXT: setp.eq.b64 %p5, %rd6, 0;
+; SM70-NEXT: selp.f64 %rd14, %rd6, %rd4, %p5;
+; SM70-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
+; SM70-NEXT: max.f64 %rd15, %rd6, %rd4;
; SM70-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
-; SM70-NEXT: setp.eq.f64 %p7, %rd5, %rd2;
+; SM70-NEXT: setp.eq.f64 %p7, %rd6, %rd4;
; SM70-NEXT: selp.f64 %rd17, %rd14, %rd16, %p7;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd9+8], %rd5, %rd17;
-; SM70-NEXT: setp.ne.b64 %p8, %rd27, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd9+8], %rd6, %rd17;
+; SM70-NEXT: setp.ne.b64 %p8, %rd27, %rd6;
; SM70-NEXT: @%p8 bra $L__BB286_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end7
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd9+16];
-; SM70-NEXT: $L__BB286_5: // %atomicrmw.start19
+; SM70-NEXT: $L__BB286_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd7, %rd28;
-; SM70-NEXT: setp.eq.b64 %p9, %rd7, 0;
-; SM70-NEXT: selp.f64 %rd18, %rd7, %rd1, %p9;
-; SM70-NEXT: setp.nan.f64 %p10, %rd7, %rd1;
-; SM70-NEXT: max.f64 %rd19, %rd7, %rd1;
+; SM70-NEXT: setp.eq.b64 %p9, %rd28, 0;
+; SM70-NEXT: selp.f64 %rd18, %rd28, %rd1, %p9;
+; SM70-NEXT: setp.nan.f64 %p10, %rd28, %rd1;
+; SM70-NEXT: max.f64 %rd19, %rd28, %rd1;
; SM70-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p10;
-; SM70-NEXT: setp.eq.f64 %p11, %rd7, %rd1;
+; SM70-NEXT: setp.eq.f64 %p11, %rd28, %rd1;
; SM70-NEXT: selp.f64 %rd21, %rd18, %rd20, %p11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd9+16], %rd7, %rd21;
-; SM70-NEXT: setp.ne.b64 %p12, %rd28, %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd28, %rd21;
+; SM70-NEXT: setp.ne.b64 %p12, %rd7, %rd28;
+; SM70-NEXT: mov.b64 %rd28, %rd7;
; SM70-NEXT: @%p12 bra $L__BB286_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end18
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd9+24];
-; SM70-NEXT: $L__BB286_7: // %atomicrmw.start26
+; SM70-NEXT: $L__BB286_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.eq.b64 %p13, %rd29, 0;
-; SM70-NEXT: selp.f64 %rd22, %rd29, %rd6, %p13;
-; SM70-NEXT: setp.nan.f64 %p14, %rd29, %rd6;
-; SM70-NEXT: max.f64 %rd23, %rd29, %rd6;
+; SM70-NEXT: selp.f64 %rd22, %rd29, %rd2, %p13;
+; SM70-NEXT: setp.nan.f64 %p14, %rd29, %rd2;
+; SM70-NEXT: max.f64 %rd23, %rd29, %rd2;
; SM70-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p14;
-; SM70-NEXT: setp.eq.f64 %p15, %rd29, %rd6;
+; SM70-NEXT: setp.eq.f64 %p15, %rd29, %rd2;
; SM70-NEXT: selp.f64 %rd25, %rd22, %rd24, %p15;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd29, %rd25;
; SM70-NEXT: setp.ne.b64 %p16, %rd8, %rd29;
; SM70-NEXT: mov.b64 %rd29, %rd8;
; SM70-NEXT: @%p16 bra $L__BB286_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
@@ -16101,132 +16071,132 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b64 {%rd10, %rd14}, [fmaximum_acq_rel_v8f64_global_cta_param_1+48];
-; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd11}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
-; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd7}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd3}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmaximum_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd50, [%rd17];
; SM70-NEXT: $L__BB287_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd50;
-; SM70-NEXT: setp.eq.b64 %p1, %rd5, 0;
-; SM70-NEXT: selp.f64 %rd18, %rd5, %rd4, %p1;
-; SM70-NEXT: setp.nan.f64 %p2, %rd5, %rd4;
-; SM70-NEXT: max.f64 %rd19, %rd5, %rd4;
+; SM70-NEXT: mov.b64 %rd7, %rd50;
+; SM70-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM70-NEXT: selp.f64 %rd18, %rd7, %rd5, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd7, %rd5;
+; SM70-NEXT: max.f64 %rd19, %rd7, %rd5;
; SM70-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p2;
-; SM70-NEXT: setp.eq.f64 %p3, %rd5, %rd4;
+; SM70-NEXT: setp.eq.f64 %p3, %rd7, %rd5;
; SM70-NEXT: selp.f64 %rd21, %rd18, %rd20, %p3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd50, [%rd17], %rd5, %rd21;
-; SM70-NEXT: setp.ne.b64 %p4, %rd50, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd50, [%rd17], %rd7, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd50, %rd7;
; SM70-NEXT: @%p4 bra $L__BB287_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd51, [%rd17+8];
-; SM70-NEXT: $L__BB287_3: // %atomicrmw.start12
+; SM70-NEXT: $L__BB287_3: // %atomicrmw.start10
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd6, %rd51;
-; SM70-NEXT: setp.eq.b64 %p5, %rd6, 0;
-; SM70-NEXT: selp.f64 %rd22, %rd6, %rd3, %p5;
-; SM70-NEXT: setp.nan.f64 %p6, %rd6, %rd3;
-; SM70-NEXT: max.f64 %rd23, %rd6, %rd3;
+; SM70-NEXT: mov.b64 %rd8, %rd51;
+; SM70-NEXT: setp.eq.b64 %p5, %rd8, 0;
+; SM70-NEXT: selp.f64 %rd22, %rd8, %rd6, %p5;
+; SM70-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
+; SM70-NEXT: max.f64 %rd23, %rd8, %rd6;
; SM70-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
-; SM70-NEXT: setp.eq.f64 %p7, %rd6, %rd3;
+; SM70-NEXT: setp.eq.f64 %p7, %rd8, %rd6;
; SM70-NEXT: selp.f64 %rd25, %rd22, %rd24, %p7;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd51, [%rd17+8], %rd6, %rd25;
-; SM70-NEXT: setp.ne.b64 %p8, %rd51, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd51, [%rd17+8], %rd8, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd51, %rd8;
; SM70-NEXT: @%p8 bra $L__BB287_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end11
+; SM70-NEXT: // %bb.4: // %atomicrmw.end9
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd52, [%rd17+16];
-; SM70-NEXT: $L__BB287_5: // %atomicrmw.start23
+; SM70-NEXT: $L__BB287_5: // %atomicrmw.start19
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd8, %rd52;
-; SM70-NEXT: setp.eq.b64 %p9, %rd8, 0;
-; SM70-NEXT: selp.f64 %rd26, %rd8, %rd2, %p9;
-; SM70-NEXT: setp.nan.f64 %p10, %rd8, %rd2;
-; SM70-NEXT: max.f64 %rd27, %rd8, %rd2;
+; SM70-NEXT: mov.b64 %rd9, %rd52;
+; SM70-NEXT: setp.eq.b64 %p9, %rd9, 0;
+; SM70-NEXT: selp.f64 %rd26, %rd9, %rd3, %p9;
+; SM70-NEXT: setp.nan.f64 %p10, %rd9, %rd3;
+; SM70-NEXT: max.f64 %rd27, %rd9, %rd3;
; SM70-NEXT: selp.f64 %rd28, 0d7FF8000000000000, %rd27, %p10;
-; SM70-NEXT: setp.eq.f64 %p11, %rd8, %rd2;
+; SM70-NEXT: setp.eq.f64 %p11, %rd9, %rd3;
; SM70-NEXT: selp.f64 %rd29, %rd26, %rd28, %p11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd52, [%rd17+16], %rd8, %rd29;
-; SM70-NEXT: setp.ne.b64 %p12, %rd52, %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd52, [%rd17+16], %rd9, %rd29;
+; SM70-NEXT: setp.ne.b64 %p12, %rd52, %rd9;
; SM70-NEXT: @%p12 bra $L__BB287_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end22
+; SM70-NEXT: // %bb.6: // %atomicrmw.end18
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd53, [%rd17+24];
-; SM70-NEXT: $L__BB287_7: // %atomicrmw.start30
+; SM70-NEXT: $L__BB287_7: // %atomicrmw.start24
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd9, %rd53;
-; SM70-NEXT: setp.eq.b64 %p13, %rd9, 0;
-; SM70-NEXT: selp.f64 %rd30, %rd9, %rd7, %p13;
-; SM70-NEXT: setp.nan.f64 %p14, %rd9, %rd7;
-; SM70-NEXT: max.f64 %rd31, %rd9, %rd7;
+; SM70-NEXT: mov.b64 %rd10, %rd53;
+; SM70-NEXT: setp.eq.b64 %p13, %rd10, 0;
+; SM70-NEXT: selp.f64 %rd30, %rd10, %rd4, %p13;
+; SM70-NEXT: setp.nan.f64 %p14, %rd10, %rd4;
+; SM70-NEXT: max.f64 %rd31, %rd10, %rd4;
; SM70-NEXT: selp.f64 %rd32, 0d7FF8000000000000, %rd31, %p14;
-; SM70-NEXT: setp.eq.f64 %p15, %rd9, %rd7;
+; SM70-NEXT: setp.eq.f64 %p15, %rd10, %rd4;
; SM70-NEXT: selp.f64 %rd33, %rd30, %rd32, %p15;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd53, [%rd17+24], %rd9, %rd33;
-; SM70-NEXT: setp.ne.b64 %p16, %rd53, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd53, [%rd17+24], %rd10, %rd33;
+; SM70-NEXT: setp.ne.b64 %p16, %rd53, %rd10;
; SM70-NEXT: @%p16 bra $L__BB287_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end29
+; SM70-NEXT: // %bb.8: // %atomicrmw.end23
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd54, [%rd17+32];
-; SM70-NEXT: $L__BB287_9: // %atomicrmw.start45
+; SM70-NEXT: $L__BB287_9: // %atomicrmw.start37
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd12, %rd54;
-; SM70-NEXT: setp.eq.b64 %p17, %rd12, 0;
-; SM70-NEXT: selp.f64 %rd34, %rd12, %rd1, %p17;
-; SM70-NEXT: setp.nan.f64 %p18, %rd12, %rd1;
-; SM70-NEXT: max.f64 %rd35, %rd12, %rd1;
+; SM70-NEXT: mov.b64 %rd13, %rd54;
+; SM70-NEXT: setp.eq.b64 %p17, %rd13, 0;
+; SM70-NEXT: selp.f64 %rd34, %rd13, %rd1, %p17;
+; SM70-NEXT: setp.nan.f64 %p18, %rd13, %rd1;
+; SM70-NEXT: max.f64 %rd35, %rd13, %rd1;
; SM70-NEXT: selp.f64 %rd36, 0d7FF8000000000000, %rd35, %p18;
-; SM70-NEXT: setp.eq.f64 %p19, %rd12, %rd1;
+; SM70-NEXT: setp.eq.f64 %p19, %rd13, %rd1;
; SM70-NEXT: selp.f64 %rd37, %rd34, %rd36, %p19;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd54, [%rd17+32], %rd12, %rd37;
-; SM70-NEXT: setp.ne.b64 %p20, %rd54, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd54, [%rd17+32], %rd13, %rd37;
+; SM70-NEXT: setp.ne.b64 %p20, %rd54, %rd13;
; SM70-NEXT: @%p20 bra $L__BB287_9;
-; SM70-NEXT: // %bb.10: // %atomicrmw.end44
+; SM70-NEXT: // %bb.10: // %atomicrmw.end36
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd55, [%rd17+40];
-; SM70-NEXT: $L__BB287_11: // %atomicrmw.start52
+; SM70-NEXT: $L__BB287_11: // %atomicrmw.start42
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd13, %rd55;
-; SM70-NEXT: setp.eq.b64 %p21, %rd13, 0;
-; SM70-NEXT: selp.f64 %rd38, %rd13, %rd11, %p21;
-; SM70-NEXT: setp.nan.f64 %p22, %rd13, %rd11;
-; SM70-NEXT: max.f64 %rd39, %rd13, %rd11;
+; SM70-NEXT: mov.b64 %rd14, %rd55;
+; SM70-NEXT: setp.eq.b64 %p21, %rd14, 0;
+; SM70-NEXT: selp.f64 %rd38, %rd14, %rd2, %p21;
+; SM70-NEXT: setp.nan.f64 %p22, %rd14, %rd2;
+; SM70-NEXT: max.f64 %rd39, %rd14, %rd2;
; SM70-NEXT: selp.f64 %rd40, 0d7FF8000000000000, %rd39, %p22;
-; SM70-NEXT: setp.eq.f64 %p23, %rd13, %rd11;
+; SM70-NEXT: setp.eq.f64 %p23, %rd14, %rd2;
; SM70-NEXT: selp.f64 %rd41, %rd38, %rd40, %p23;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd55, [%rd17+40], %rd13, %rd41;
-; SM70-NEXT: setp.ne.b64 %p24, %rd55, %rd13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd55, [%rd17+40], %rd14, %rd41;
+; SM70-NEXT: setp.ne.b64 %p24, %rd55, %rd14;
; SM70-NEXT: @%p24 bra $L__BB287_11;
-; SM70-NEXT: // %bb.12: // %atomicrmw.end51
+; SM70-NEXT: // %bb.12: // %atomicrmw.end41
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd56, [%rd17+48];
-; SM70-NEXT: $L__BB287_13: // %atomicrmw.start63
+; SM70-NEXT: $L__BB287_13: // %atomicrmw.start51
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd15, %rd56;
-; SM70-NEXT: setp.eq.b64 %p25, %rd15, 0;
-; SM70-NEXT: selp.f64 %rd42, %rd15, %rd10, %p25;
-; SM70-NEXT: setp.nan.f64 %p26, %rd15, %rd10;
-; SM70-NEXT: max.f64 %rd43, %rd15, %rd10;
+; SM70-NEXT: setp.eq.b64 %p25, %rd56, 0;
+; SM70-NEXT: selp.f64 %rd42, %rd56, %rd11, %p25;
+; SM70-NEXT: setp.nan.f64 %p26, %rd56, %rd11;
+; SM70-NEXT: max.f64 %rd43, %rd56, %rd11;
; SM70-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
-; SM70-NEXT: setp.eq.f64 %p27, %rd15, %rd10;
+; SM70-NEXT: setp.eq.f64 %p27, %rd56, %rd11;
; SM70-NEXT: selp.f64 %rd45, %rd42, %rd44, %p27;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd56, [%rd17+48], %rd15, %rd45;
-; SM70-NEXT: setp.ne.b64 %p28, %rd56, %rd15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd56, %rd45;
+; SM70-NEXT: setp.ne.b64 %p28, %rd15, %rd56;
+; SM70-NEXT: mov.b64 %rd56, %rd15;
; SM70-NEXT: @%p28 bra $L__BB287_13;
-; SM70-NEXT: // %bb.14: // %atomicrmw.end62
+; SM70-NEXT: // %bb.14: // %atomicrmw.end50
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd57, [%rd17+56];
-; SM70-NEXT: $L__BB287_15: // %atomicrmw.start70
+; SM70-NEXT: $L__BB287_15: // %atomicrmw.start56
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.eq.b64 %p29, %rd57, 0;
-; SM70-NEXT: selp.f64 %rd46, %rd57, %rd14, %p29;
-; SM70-NEXT: setp.nan.f64 %p30, %rd57, %rd14;
-; SM70-NEXT: max.f64 %rd47, %rd57, %rd14;
+; SM70-NEXT: selp.f64 %rd46, %rd57, %rd12, %p29;
+; SM70-NEXT: setp.nan.f64 %p30, %rd57, %rd12;
+; SM70-NEXT: max.f64 %rd47, %rd57, %rd12;
; SM70-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p30;
-; SM70-NEXT: setp.eq.f64 %p31, %rd57, %rd14;
+; SM70-NEXT: setp.eq.f64 %p31, %rd57, %rd12;
; SM70-NEXT: selp.f64 %rd49, %rd46, %rd48, %p31;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd57, %rd49;
; SM70-NEXT: setp.ne.b64 %p32, %rd16, %rd57;
; SM70-NEXT: mov.b64 %rd57, %rd16;
; SM70-NEXT: @%p32 bra $L__BB287_15;
-; SM70-NEXT: // %bb.16: // %atomicrmw.end69
+; SM70-NEXT: // %bb.16: // %atomicrmw.end55
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd56, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd54, %rd55};
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
@@ -16243,10 +16213,8 @@ define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f16_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
; SM70-NEXT: st.param.b16 [func_retval0], %rs2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
@@ -16256,17 +16224,14 @@ define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-LABEL: fadd_acq_rel_v2f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<3>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2f16_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r2, [%rd1], %r1;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -16275,19 +16240,17 @@ define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM70-LABEL: fadd_acq_rel_v4f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<5>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4f16_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs5, [%rd1], %rs1;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs2;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs3;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs4;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r4, [%rd1+4], %r2;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -16296,7 +16259,6 @@ define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM70-LABEL: fadd_acq_rel_v8f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .b16 %rs<17>;
; SM70-NEXT: .reg .b32 %r<9>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
@@ -16304,22 +16266,10 @@ define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f16_global_cta_param_1];
-; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs9, [%rd1], %rs7;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+2], %rs8;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+4], %rs5;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs6;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs13, [%rd1+8], %rs3;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+10], %rs4;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs1;
-; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs2;
-; SM70-NEXT: mov.b32 %r5, {%rs9, %rs10};
-; SM70-NEXT: mov.b32 %r6, {%rs11, %rs12};
-; SM70-NEXT: mov.b32 %r7, {%rs13, %rs14};
-; SM70-NEXT: mov.b32 %r8, {%rs15, %rs16};
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r8, [%rd1+12], %r4;
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT: ret;
@@ -16335,11 +16285,9 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f16_global_cta_param_0];
-; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs2, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %rs3;
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
@@ -16350,11 +16298,9 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f16_global_cta_param_0];
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs2, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %rs3;
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
@@ -16365,11 +16311,9 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f16_global_cta_param_0];
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs2, %rs1;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %rs3;
; SM70-FTZ-DISALLOW-NEXT: ret;
;
@@ -16380,11 +16324,9 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f16_global_cta_param_0];
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs2, %rs1;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %rs3;
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
@@ -16394,70 +16336,54 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-DISALLOW: {
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<4>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
-; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fsub_acq_rel_v2f16_global_cta_param_1];
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs3, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1], %rs3;
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs5, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs5;
-; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs6};
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r2, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r3;
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-ALLOW: {
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<4>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fsub_acq_rel_v2f16_global_cta_param_1];
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs3, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1], %rs3;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs5, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs5;
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs6};
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: neg.f16x2 %r2, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r3;
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM70-FTZ-DISALLOW: {
-; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<4>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fsub_acq_rel_v2f16_global_cta_param_1];
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs3, %rs1;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1], %rs3;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs5, %rs2;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs5;
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs6};
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r2, %r1;
+; SM70-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r3;
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM70-FTZ-ALLOW: {
-; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<4>;
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fsub_acq_rel_v2f16_global_cta_param_1];
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs3, %rs1;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1], %rs3;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs5, %rs2;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs5;
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs6};
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r2, %r1;
+; SM70-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r3;
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -16466,86 +16392,70 @@ define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-DISALLOW: {
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<13>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r3, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r4, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fsub_acq_rel_v4f16_global_cta_param_1];
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs5, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1], %rs5;
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs7, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+2], %rs7;
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs9, %rs3;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+4], %rs9;
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs11, %rs4;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs11;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r5, [%rd1], %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r6, [%rd1+4], %r3;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.v4.b16 [func_retval0], {%rs6, %rs8, %rs10, %rs12};
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-ALLOW: {
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<13>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<7>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: neg.f16x2 %r3, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f16x2 %r4, %r1;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fsub_acq_rel_v4f16_global_cta_param_1];
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs5, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1], %rs5;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs7, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+2], %rs7;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs9, %rs3;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+4], %rs9;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs11, %rs4;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs11;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r5, [%rd1], %r4;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r6, [%rd1+4], %r3;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.v4.b16 [func_retval0], {%rs6, %rs8, %rs10, %rs12};
+; SM70-NOFTZ-ALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM70-FTZ-DISALLOW: {
-; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<13>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r3, %r2;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r4, %r1;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fsub_acq_rel_v4f16_global_cta_param_1];
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs5, %rs1;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1], %rs5;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs7, %rs2;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+2], %rs7;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs9, %rs3;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+4], %rs9;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs11, %rs4;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs11;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r5, [%rd1], %r4;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r6, [%rd1+4], %r3;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.v4.b16 [func_retval0], {%rs6, %rs8, %rs10, %rs12};
+; SM70-FTZ-DISALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM70-FTZ-ALLOW: {
-; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<13>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<7>;
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r3, %r2;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r4, %r1;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fsub_acq_rel_v4f16_global_cta_param_1];
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs5, %rs1;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1], %rs5;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs7, %rs2;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+2], %rs7;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs9, %rs3;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+4], %rs9;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs11, %rs4;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs11;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r5, [%rd1], %r4;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r6, [%rd1+4], %r3;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.v4.b16 [func_retval0], {%rs6, %rs8, %rs10, %rs12};
+; SM70-FTZ-ALLOW-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -16554,154 +16464,86 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-DISALLOW: {
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<25>;
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<9>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
-; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs9, %rs7;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1], %rs9;
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs11, %rs8;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+2], %rs11;
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs13, %rs5;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+4], %rs13;
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs15, %rs6;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+6], %rs15;
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs17, %rs3;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs18, [%rd1+8], %rs17;
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs19, %rs4;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs20, [%rd1+10], %rs19;
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs21, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs22, [%rd1+12], %rs21;
-; SM70-NOFTZ-DISALLOW-NEXT: neg.f16 %rs23, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs24, [%rd1+14], %rs23;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r5, {%rs10, %rs12};
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, {%rs14, %rs16};
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, {%rs18, %rs20};
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r8, {%rs22, %rs24};
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r5, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r6, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r7, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r8, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r9, [%rd1], %r8;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r10, [%rd1+4], %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r11, [%rd1+8], %r6;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r12, [%rd1+12], %r5;
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-ALLOW: {
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<25>;
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<9>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<13>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
-; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs9, %rs7;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1], %rs9;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs11, %rs8;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+2], %rs11;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs13, %rs5;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+4], %rs13;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs15, %rs6;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+6], %rs15;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs17, %rs3;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs18, [%rd1+8], %rs17;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs19, %rs4;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs20, [%rd1+10], %rs19;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs21, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs22, [%rd1+12], %rs21;
-; SM70-NOFTZ-ALLOW-NEXT: neg.f16 %rs23, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs24, [%rd1+14], %rs23;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r5, {%rs10, %rs12};
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, {%rs14, %rs16};
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r7, {%rs18, %rs20};
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r8, {%rs22, %rs24};
+; SM70-NOFTZ-ALLOW-NEXT: neg.f16x2 %r5, %r4;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f16x2 %r6, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f16x2 %r7, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: neg.f16x2 %r8, %r1;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r9, [%rd1], %r8;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r10, [%rd1+4], %r7;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r11, [%rd1+8], %r6;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r12, [%rd1+12], %r5;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM70-FTZ-DISALLOW: {
-; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<25>;
-; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<9>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs9, %rs7;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1], %rs9;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs11, %rs8;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+2], %rs11;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs13, %rs5;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+4], %rs13;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs15, %rs6;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+6], %rs15;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs17, %rs3;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs18, [%rd1+8], %rs17;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs19, %rs4;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs20, [%rd1+10], %rs19;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs21, %rs1;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs22, [%rd1+12], %rs21;
-; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs23, %rs2;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs24, [%rd1+14], %rs23;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r5, {%rs10, %rs12};
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, {%rs14, %rs16};
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r7, {%rs18, %rs20};
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r8, {%rs22, %rs24};
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r5, %r4;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r6, %r3;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r7, %r2;
+; SM70-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r8, %r1;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r9, [%rd1], %r8;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r10, [%rd1+4], %r7;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r11, [%rd1+8], %r6;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r12, [%rd1+12], %r5;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM70-FTZ-ALLOW: {
-; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<25>;
-; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<9>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<13>;
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
-; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs9, %rs7;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1], %rs9;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs11, %rs8;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+2], %rs11;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs13, %rs5;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+4], %rs13;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs15, %rs6;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+6], %rs15;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs17, %rs3;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs18, [%rd1+8], %rs17;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs19, %rs4;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs20, [%rd1+10], %rs19;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs21, %rs1;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs22, [%rd1+12], %rs21;
-; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs23, %rs2;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs24, [%rd1+14], %rs23;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r5, {%rs10, %rs12};
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, {%rs14, %rs16};
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r7, {%rs18, %rs20};
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r8, {%rs22, %rs24};
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r5, %r4;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r6, %r3;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r7, %r2;
+; SM70-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r8, %r1;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r9, [%rd1], %r8;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r10, [%rd1+4], %r7;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r11, [%rd1+8], %r6;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16x2 %r12, [%rd1+12], %r5;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
ret <8 x half> %retval
@@ -16716,9 +16558,9 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -16756,9 +16598,9 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -16796,9 +16638,9 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -16836,9 +16678,9 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -17640,9 +17482,9 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -17680,9 +17522,9 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -17720,9 +17562,9 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -17760,9 +17602,9 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -18564,9 +18406,9 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -18608,9 +18450,9 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -18652,9 +18494,9 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -18696,9 +18538,9 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -19728,9 +19570,9 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -19772,9 +19614,9 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -19816,9 +19658,9 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -19860,9 +19702,9 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -20892,9 +20734,9 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -20938,9 +20780,9 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -20984,9 +20826,9 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -21030,9 +20872,9 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -22288,9 +22130,9 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -22334,9 +22176,9 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -22380,9 +22222,9 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -22426,9 +22268,9 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -23032,644 +22874,660 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NOFTZ-DISALLOW: {
; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<11>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<17>;
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<101>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<105>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: xor.b32 %r2, %r6, -2147450880;
+; SM70-NOFTZ-DISALLOW-NEXT: xor.b32 %r1, %r5, -2147450880;
+; SM70-NOFTZ-DISALLOW-NEXT: xor.b32 %r4, %r8, -2147450880;
+; SM70-NOFTZ-DISALLOW-NEXT: xor.b32 %r3, %r7, -2147450880;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r97, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r102}, %rd2; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r101, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB319_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r97;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r7, %rs4;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r8, %r7, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r9, %r8, %r6;
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r10, %r9, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r11, %r10, 1;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r12, %r11, %r9;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r13, %r12, 32767;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r9, %r9;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r14, %r9, 4194304;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r16, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r17, %r16, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r18, %rs3;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r20, %r19, %r17;
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r21, %r20, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r22, %r21, 1;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r23, %r22, %r20;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r24, %r23, 32767;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r20, %r20;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r25, %r20, 4194304;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r26, %r25, %r24, %p2;
-; SM70-NOFTZ-DISALLOW-NEXT: prmt.b32 %r27, %r26, %r15, 0x7632U;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r27;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r28, %rs6;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r29, %r28, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r98;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r30, %rs8;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r31, %r30, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r32, %r31, %r29;
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r33, %r32, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r34, %r33, 1;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r35, %r34, %r32;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r36, %r35, 32767;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p3, %r32, %r32;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r37, %r32, 4194304;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r38, %r37, %r36, %p3;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r39, %rs5;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r40, %r39, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r41, %rs7;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r42, %r41, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r43, %r42, %r40;
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r44, %r43, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r45, %r44, 1;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r46, %r45, %r43;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r47, %r46, 32767;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p4, %r43, %r43;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r48, %r43, 4194304;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r49, %r48, %r47, %p4;
-; SM70-NOFTZ-DISALLOW-NEXT: prmt.b32 %r50, %r49, %r38, 0x7632U;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r101;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r11, %rs4;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r13, %r12, %r10;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r14, %r13, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r15, %r14, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r16, %r15, %r13;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r17, %r16, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r13, %r13;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r18, %r13, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r19, %r18, %r17, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r20, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r21, %r20, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r22, %rs3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r23, %r22, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r24, %r23, %r21;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r25, %r24, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r26, %r25, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r27, %r26, %r24;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r28, %r27, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r24, %r24;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r29, %r24, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r30, %r29, %r28, %p2;
+; SM70-NOFTZ-DISALLOW-NEXT: prmt.b32 %r31, %r30, %r19, 0x7632U;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r31;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r32, %rs6;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r33, %r32, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r102;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r34, %rs8;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r36, %r35, %r33;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r37, %r36, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r38, %r37, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r39, %r38, %r36;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r40, %r39, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p3, %r36, %r36;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r41, %r36, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r42, %r41, %r40, %p3;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r43, %rs5;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r44, %r43, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r45, %rs7;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r46, %r45, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r47, %r46, %r44;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r48, %r47, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r49, %r48, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r50, %r49, %r47;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r51, %r50, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p4, %r47, %r47;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r52, %r47, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r53, %r52, %r51, %p4;
+; SM70-NOFTZ-DISALLOW-NEXT: prmt.b32 %r54, %r53, %r42, 0x7632U;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r54;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r97;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r98;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r101;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r102;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r97, %rd11;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r102}, %rd11; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r101, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT: @%p5 bra $L__BB319_1;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r99, %rd12;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r104}, %rd12; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r103, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB319_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r51, %rs10;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r52, %r51, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r99;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r53, %rs12;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r54, %r53, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r55, %r54, %r52;
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r56, %r55, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r57, %r56, 1;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r58, %r57, %r55;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r59, %r58, 32767;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p6, %r55, %r55;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r60, %r55, 4194304;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r61, %r60, %r59, %p6;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r62, %rs9;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r63, %r62, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r64, %rs11;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r65, %r64, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r66, %r65, %r63;
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r67, %r66, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r68, %r67, 1;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r69, %r68, %r66;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r70, %r69, 32767;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p7, %r66, %r66;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r71, %r66, 4194304;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r72, %r71, %r70, %p7;
-; SM70-NOFTZ-DISALLOW-NEXT: prmt.b32 %r73, %r72, %r61, 0x7632U;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r73;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r74, %rs14;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r75, %r74, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r100;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r76, %rs16;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r77, %r76, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r78, %r77, %r75;
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r79, %r78, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r80, %r79, 1;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r81, %r80, %r78;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r82, %r81, 32767;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p8, %r78, %r78;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r83, %r78, 4194304;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r84, %r83, %r82, %p8;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r85, %rs13;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r86, %r85, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r87, %rs15;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r88, %r87, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r89, %r88, %r86;
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r90, %r89, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r91, %r90, 1;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r92, %r91, %r89;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r93, %r92, 32767;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p9, %r89, %r89;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r94, %r89, 4194304;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r95, %r94, %r93, %p9;
-; SM70-NOFTZ-DISALLOW-NEXT: prmt.b32 %r96, %r95, %r84, 0x7632U;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r96;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r55, %rs10;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r56, %r55, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r103;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r57, %rs12;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r58, %r57, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r59, %r58, %r56;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r60, %r59, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r61, %r60, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r62, %r61, %r59;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r63, %r62, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p6, %r59, %r59;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r64, %r59, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r65, %r64, %r63, %p6;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r66, %rs9;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r67, %r66, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r68, %rs11;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r69, %r68, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r70, %r69, %r67;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r71, %r70, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r72, %r71, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r73, %r72, %r70;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r74, %r73, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p7, %r70, %r70;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r75, %r70, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r76, %r75, %r74, %p7;
+; SM70-NOFTZ-DISALLOW-NEXT: prmt.b32 %r77, %r76, %r65, 0x7632U;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r77;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r78, %rs14;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r79, %r78, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r104;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r80, %rs16;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r81, %r80, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r82, %r81, %r79;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r83, %r82, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r84, %r83, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r85, %r84, %r82;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r86, %r85, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p8, %r82, %r82;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r87, %r82, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r88, %r87, %r86, %p8;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r89, %rs13;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r90, %r89, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r91, %rs15;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r92, %r91, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r93, %r92, %r90;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r94, %r93, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r95, %r94, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r96, %r95, %r93;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r97, %r96, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p9, %r93, %r93;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r98, %r93, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r99, %r98, %r97, %p9;
+; SM70-NOFTZ-DISALLOW-NEXT: prmt.b32 %r100, %r99, %r88, 0x7632U;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r100;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r99;
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r100;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r103;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r104;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
-; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
-; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r99, %rd21;
+; SM70-NOFTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r104}, %rd21; }
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r103, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT: @%p10 bra $L__BB319_3;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r101, %r102, %r103, %r104};
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v8bf16_global_cta(
; SM70-NOFTZ-ALLOW: {
; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<11>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<17>;
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<101>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<105>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: xor.b32 %r2, %r6, -2147450880;
+; SM70-NOFTZ-ALLOW-NEXT: xor.b32 %r1, %r5, -2147450880;
+; SM70-NOFTZ-ALLOW-NEXT: xor.b32 %r4, %r8, -2147450880;
+; SM70-NOFTZ-ALLOW-NEXT: xor.b32 %r3, %r7, -2147450880;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r97, %rd2;
+; SM70-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r102}, %rd2; }
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r101, %rd2;
; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-ALLOW-NEXT: $L__BB319_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r97;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r7, %rs4;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r8, %r7, 16;
-; SM70-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r9, %r8, %r6;
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r10, %r9, 16;
-; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r11, %r10, 1;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r12, %r11, %r9;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r13, %r12, 32767;
-; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r9, %r9;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r14, %r9, 4194304;
-; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r16, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r17, %r16, 16;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r18, %rs3;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r19, %r18, 16;
-; SM70-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r20, %r19, %r17;
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r21, %r20, 16;
-; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r22, %r21, 1;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r23, %r22, %r20;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r24, %r23, 32767;
-; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r20, %r20;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r25, %r20, 4194304;
-; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r26, %r25, %r24, %p2;
-; SM70-NOFTZ-ALLOW-NEXT: prmt.b32 %r27, %r26, %r15, 0x7632U;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r27;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r28, %rs6;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r29, %r28, 16;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r98;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r30, %rs8;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r31, %r30, 16;
-; SM70-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r32, %r31, %r29;
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r33, %r32, 16;
-; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r34, %r33, 1;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r35, %r34, %r32;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r36, %r35, 32767;
-; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p3, %r32, %r32;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r37, %r32, 4194304;
-; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r38, %r37, %r36, %p3;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r39, %rs5;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r40, %r39, 16;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r41, %rs7;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r42, %r41, 16;
-; SM70-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r43, %r42, %r40;
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r44, %r43, 16;
-; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r45, %r44, 1;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r46, %r45, %r43;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r47, %r46, 32767;
-; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p4, %r43, %r43;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r48, %r43, 4194304;
-; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r49, %r48, %r47, %p4;
-; SM70-NOFTZ-ALLOW-NEXT: prmt.b32 %r50, %r49, %r38, 0x7632U;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs2;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r101;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r11, %rs4;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NOFTZ-ALLOW-NEXT: add.rn.f32 %r13, %r12, %r10;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r14, %r13, 16;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r15, %r14, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r16, %r15, %r13;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r17, %r16, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r13, %r13;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r18, %r13, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r19, %r18, %r17, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r20, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r21, %r20, 16;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r22, %rs3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r23, %r22, 16;
+; SM70-NOFTZ-ALLOW-NEXT: add.rn.f32 %r24, %r23, %r21;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r25, %r24, 16;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r26, %r25, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r27, %r26, %r24;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r28, %r27, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r24, %r24;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r29, %r24, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r30, %r29, %r28, %p2;
+; SM70-NOFTZ-ALLOW-NEXT: prmt.b32 %r31, %r30, %r19, 0x7632U;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r31;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r32, %rs6;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r33, %r32, 16;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r102;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r34, %rs8;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-NOFTZ-ALLOW-NEXT: add.rn.f32 %r36, %r35, %r33;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r37, %r36, 16;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r38, %r37, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r39, %r38, %r36;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r40, %r39, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p3, %r36, %r36;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r41, %r36, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r42, %r41, %r40, %p3;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r43, %rs5;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r44, %r43, 16;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r45, %rs7;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r46, %r45, 16;
+; SM70-NOFTZ-ALLOW-NEXT: add.rn.f32 %r47, %r46, %r44;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r48, %r47, 16;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r49, %r48, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r50, %r49, %r47;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r51, %r50, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p4, %r47, %r47;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r52, %r47, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r53, %r52, %r51, %p4;
+; SM70-NOFTZ-ALLOW-NEXT: prmt.b32 %r54, %r53, %r42, 0x7632U;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r54;
; SM70-NOFTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r97;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r98;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r101;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r102;
; SM70-NOFTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
-; SM70-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r97, %rd11;
+; SM70-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r102}, %rd11; }
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r101, %rd11;
; SM70-NOFTZ-ALLOW-NEXT: @%p5 bra $L__BB319_1;
; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
-; SM70-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r99, %rd12;
+; SM70-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r104}, %rd12; }
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r103, %rd12;
; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r4;
; SM70-NOFTZ-ALLOW-NEXT: $L__BB319_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r51, %rs10;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r52, %r51, 16;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r99;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r53, %rs12;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r54, %r53, 16;
-; SM70-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r55, %r54, %r52;
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r56, %r55, 16;
-; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r57, %r56, 1;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r58, %r57, %r55;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r59, %r58, 32767;
-; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p6, %r55, %r55;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r60, %r55, 4194304;
-; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r61, %r60, %r59, %p6;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r62, %rs9;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r63, %r62, 16;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r64, %rs11;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r65, %r64, 16;
-; SM70-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r66, %r65, %r63;
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r67, %r66, 16;
-; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r68, %r67, 1;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r69, %r68, %r66;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r70, %r69, 32767;
-; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p7, %r66, %r66;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r71, %r66, 4194304;
-; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r72, %r71, %r70, %p7;
-; SM70-NOFTZ-ALLOW-NEXT: prmt.b32 %r73, %r72, %r61, 0x7632U;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r73;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r74, %rs14;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r75, %r74, 16;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r100;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r76, %rs16;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r77, %r76, 16;
-; SM70-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r78, %r77, %r75;
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r79, %r78, 16;
-; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r80, %r79, 1;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r81, %r80, %r78;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r82, %r81, 32767;
-; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p8, %r78, %r78;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r83, %r78, 4194304;
-; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r84, %r83, %r82, %p8;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r85, %rs13;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r86, %r85, 16;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r87, %rs15;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r88, %r87, 16;
-; SM70-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r89, %r88, %r86;
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r90, %r89, 16;
-; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r91, %r90, 1;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r92, %r91, %r89;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r93, %r92, 32767;
-; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p9, %r89, %r89;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r94, %r89, 4194304;
-; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r95, %r94, %r93, %p9;
-; SM70-NOFTZ-ALLOW-NEXT: prmt.b32 %r96, %r95, %r84, 0x7632U;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r96;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r55, %rs10;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r56, %r55, 16;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r103;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r57, %rs12;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r58, %r57, 16;
+; SM70-NOFTZ-ALLOW-NEXT: add.rn.f32 %r59, %r58, %r56;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r60, %r59, 16;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r61, %r60, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r62, %r61, %r59;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r63, %r62, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p6, %r59, %r59;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r64, %r59, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r65, %r64, %r63, %p6;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r66, %rs9;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r67, %r66, 16;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r68, %rs11;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r69, %r68, 16;
+; SM70-NOFTZ-ALLOW-NEXT: add.rn.f32 %r70, %r69, %r67;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r71, %r70, 16;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r72, %r71, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r73, %r72, %r70;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r74, %r73, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p7, %r70, %r70;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r75, %r70, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r76, %r75, %r74, %p7;
+; SM70-NOFTZ-ALLOW-NEXT: prmt.b32 %r77, %r76, %r65, 0x7632U;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r77;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r78, %rs14;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r79, %r78, 16;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r104;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r80, %rs16;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r81, %r80, 16;
+; SM70-NOFTZ-ALLOW-NEXT: add.rn.f32 %r82, %r81, %r79;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r83, %r82, 16;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r84, %r83, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r85, %r84, %r82;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r86, %r85, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p8, %r82, %r82;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r87, %r82, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r88, %r87, %r86, %p8;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r89, %rs13;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r90, %r89, 16;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r91, %rs15;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r92, %r91, 16;
+; SM70-NOFTZ-ALLOW-NEXT: add.rn.f32 %r93, %r92, %r90;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r94, %r93, 16;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r95, %r94, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r96, %r95, %r93;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r97, %r96, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p9, %r93, %r93;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r98, %r93, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r99, %r98, %r97, %p9;
+; SM70-NOFTZ-ALLOW-NEXT: prmt.b32 %r100, %r99, %r88, 0x7632U;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r100;
; SM70-NOFTZ-ALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r99;
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r100;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r103;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r104;
; SM70-NOFTZ-ALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
-; SM70-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
-; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r99, %rd21;
+; SM70-NOFTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r104}, %rd21; }
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r103, %rd21;
; SM70-NOFTZ-ALLOW-NEXT: @%p10 bra $L__BB319_3;
; SM70-NOFTZ-ALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
+; SM70-NOFTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r101, %r102, %r103, %r104};
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8bf16_global_cta(
; SM70-FTZ-DISALLOW: {
; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<11>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<17>;
-; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<101>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<105>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<22>;
-; SM70-FTZ-DISALLOW-EMPTY:
-; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
-; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
-; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r97, %rd2;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM70-FTZ-DISALLOW-NEXT: $L__BB319_1: // %atomicrmw.start
-; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r97;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r7, %rs4;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r8, %r7, 16;
-; SM70-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r9, %r8, %r6;
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r10, %r9, 16;
-; SM70-FTZ-DISALLOW-NEXT: and.b32 %r11, %r10, 1;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r12, %r11, %r9;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r13, %r12, 32767;
-; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r9, %r9;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r14, %r9, 4194304;
-; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r16, %rs1;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r17, %r16, 16;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r18, %rs3;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, 16;
-; SM70-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r20, %r19, %r17;
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r21, %r20, 16;
-; SM70-FTZ-DISALLOW-NEXT: and.b32 %r22, %r21, 1;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r23, %r22, %r20;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r24, %r23, 32767;
-; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r20, %r20;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r25, %r20, 4194304;
-; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r26, %r25, %r24, %p2;
-; SM70-FTZ-DISALLOW-NEXT: prmt.b32 %r27, %r26, %r15, 0x7632U;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r27;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r28, %rs6;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r29, %r28, 16;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r98;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r30, %rs8;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r31, %r30, 16;
-; SM70-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r32, %r31, %r29;
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r33, %r32, 16;
-; SM70-FTZ-DISALLOW-NEXT: and.b32 %r34, %r33, 1;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r35, %r34, %r32;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r36, %r35, 32767;
-; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p3, %r32, %r32;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r37, %r32, 4194304;
-; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r38, %r37, %r36, %p3;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r39, %rs5;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r40, %r39, 16;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r41, %rs7;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r42, %r41, 16;
-; SM70-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r43, %r42, %r40;
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r44, %r43, 16;
-; SM70-FTZ-DISALLOW-NEXT: and.b32 %r45, %r44, 1;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r46, %r45, %r43;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r47, %r46, 32767;
-; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p4, %r43, %r43;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r48, %r43, 4194304;
-; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r49, %r48, %r47, %p4;
-; SM70-FTZ-DISALLOW-NEXT: prmt.b32 %r50, %r49, %r38, 0x7632U;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r50;
-; SM70-FTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-FTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r97;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r98;
-; SM70-FTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-FTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-FTZ-DISALLOW-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
-; SM70-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r97, %rd11;
-; SM70-FTZ-DISALLOW-NEXT: @%p5 bra $L__BB319_1;
-; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
-; SM70-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r99, %rd12;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r3;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r4;
-; SM70-FTZ-DISALLOW-NEXT: $L__BB319_3: // %atomicrmw.start2
-; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r51, %rs10;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r52, %r51, 16;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r99;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r53, %rs12;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r54, %r53, 16;
-; SM70-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r55, %r54, %r52;
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r56, %r55, 16;
-; SM70-FTZ-DISALLOW-NEXT: and.b32 %r57, %r56, 1;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r58, %r57, %r55;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r59, %r58, 32767;
-; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p6, %r55, %r55;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r60, %r55, 4194304;
-; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r61, %r60, %r59, %p6;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r62, %rs9;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r63, %r62, 16;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r64, %rs11;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r65, %r64, 16;
-; SM70-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r66, %r65, %r63;
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r67, %r66, 16;
-; SM70-FTZ-DISALLOW-NEXT: and.b32 %r68, %r67, 1;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r69, %r68, %r66;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r70, %r69, 32767;
-; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p7, %r66, %r66;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r71, %r66, 4194304;
-; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r72, %r71, %r70, %p7;
-; SM70-FTZ-DISALLOW-NEXT: prmt.b32 %r73, %r72, %r61, 0x7632U;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r73;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r74, %rs14;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r75, %r74, 16;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r100;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r76, %rs16;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r77, %r76, 16;
-; SM70-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r78, %r77, %r75;
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r79, %r78, 16;
-; SM70-FTZ-DISALLOW-NEXT: and.b32 %r80, %r79, 1;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r81, %r80, %r78;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r82, %r81, 32767;
-; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p8, %r78, %r78;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r83, %r78, 4194304;
-; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r84, %r83, %r82, %p8;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r85, %rs13;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r86, %r85, 16;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r87, %rs15;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r88, %r87, 16;
-; SM70-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r89, %r88, %r86;
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r90, %r89, 16;
-; SM70-FTZ-DISALLOW-NEXT: and.b32 %r91, %r90, 1;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r92, %r91, %r89;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r93, %r92, 32767;
-; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p9, %r89, %r89;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r94, %r89, 4194304;
-; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r95, %r94, %r93, %p9;
-; SM70-FTZ-DISALLOW-NEXT: prmt.b32 %r96, %r95, %r84, 0x7632U;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r96;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: xor.b32 %r2, %r6, -2147450880;
+; SM70-FTZ-DISALLOW-NEXT: xor.b32 %r1, %r5, -2147450880;
+; SM70-FTZ-DISALLOW-NEXT: xor.b32 %r4, %r8, -2147450880;
+; SM70-FTZ-DISALLOW-NEXT: xor.b32 %r3, %r7, -2147450880;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r102}, %rd2; }
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r101, %rd2;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-FTZ-DISALLOW-NEXT: $L__BB319_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs2;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r101;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r11, %rs4;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r13, %r12, %r10;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r14, %r13, 16;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r15, %r14, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r16, %r15, %r13;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r17, %r16, 32767;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r13, %r13;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r18, %r13, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r19, %r18, %r17, %p1;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r20, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r21, %r20, 16;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r22, %rs3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r23, %r22, 16;
+; SM70-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r24, %r23, %r21;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r25, %r24, 16;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r26, %r25, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r27, %r26, %r24;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r28, %r27, 32767;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r24, %r24;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r29, %r24, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r30, %r29, %r28, %p2;
+; SM70-FTZ-DISALLOW-NEXT: prmt.b32 %r31, %r30, %r19, 0x7632U;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r31;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r32, %rs6;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r33, %r32, 16;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r102;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r34, %rs8;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r36, %r35, %r33;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r37, %r36, 16;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r38, %r37, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r39, %r38, %r36;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r40, %r39, 32767;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p3, %r36, %r36;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r41, %r36, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r42, %r41, %r40, %p3;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r43, %rs5;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r44, %r43, 16;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r45, %rs7;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r46, %r45, 16;
+; SM70-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r47, %r46, %r44;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r48, %r47, 16;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r49, %r48, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r50, %r49, %r47;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r51, %r50, 32767;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p4, %r47, %r47;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r52, %r47, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r53, %r52, %r51, %p4;
+; SM70-FTZ-DISALLOW-NEXT: prmt.b32 %r54, %r53, %r42, 0x7632U;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r54;
+; SM70-FTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-FTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r101;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r102;
+; SM70-FTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-FTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r102}, %rd11; }
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r101, %rd11;
+; SM70-FTZ-DISALLOW-NEXT: @%p5 bra $L__BB319_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r104}, %rd12; }
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r103, %rd12;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM70-FTZ-DISALLOW-NEXT: $L__BB319_3: // %atomicrmw.start2
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r55, %rs10;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r56, %r55, 16;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r103;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r57, %rs12;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r58, %r57, 16;
+; SM70-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r59, %r58, %r56;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r60, %r59, 16;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r61, %r60, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r62, %r61, %r59;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r63, %r62, 32767;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p6, %r59, %r59;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r64, %r59, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r65, %r64, %r63, %p6;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r66, %rs9;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r67, %r66, 16;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r68, %rs11;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r69, %r68, 16;
+; SM70-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r70, %r69, %r67;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r71, %r70, 16;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r72, %r71, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r73, %r72, %r70;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r74, %r73, 32767;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p7, %r70, %r70;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r75, %r70, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r76, %r75, %r74, %p7;
+; SM70-FTZ-DISALLOW-NEXT: prmt.b32 %r77, %r76, %r65, 0x7632U;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r77;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r78, %rs14;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r79, %r78, 16;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r104;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r80, %rs16;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r81, %r80, 16;
+; SM70-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r82, %r81, %r79;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r83, %r82, 16;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r84, %r83, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r85, %r84, %r82;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r86, %r85, 32767;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p8, %r82, %r82;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r87, %r82, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r88, %r87, %r86, %p8;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r89, %rs13;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r90, %r89, 16;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r91, %rs15;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r92, %r91, 16;
+; SM70-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r93, %r92, %r90;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r94, %r93, 16;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r95, %r94, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r96, %r95, %r93;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r97, %r96, 32767;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p9, %r93, %r93;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r98, %r93, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r99, %r98, %r97, %p9;
+; SM70-FTZ-DISALLOW-NEXT: prmt.b32 %r100, %r99, %r88, 0x7632U;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r100;
; SM70-FTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r99;
-; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r100;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r103;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r104;
; SM70-FTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
-; SM70-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
-; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r99, %rd21;
+; SM70-FTZ-DISALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r104}, %rd21; }
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r103, %rd21;
; SM70-FTZ-DISALLOW-NEXT: @%p10 bra $L__BB319_3;
; SM70-FTZ-DISALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
+; SM70-FTZ-DISALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r101, %r102, %r103, %r104};
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v8bf16_global_cta(
; SM70-FTZ-ALLOW: {
; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<11>;
; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<17>;
-; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<101>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<105>;
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: xor.b32 %r2, %r6, -2147450880;
+; SM70-FTZ-ALLOW-NEXT: xor.b32 %r1, %r5, -2147450880;
+; SM70-FTZ-ALLOW-NEXT: xor.b32 %r4, %r8, -2147450880;
+; SM70-FTZ-ALLOW-NEXT: xor.b32 %r3, %r7, -2147450880;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r97, %rd2;
+; SM70-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r102}, %rd2; }
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r101, %rd2;
; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-ALLOW-NEXT: $L__BB319_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r97;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r7, %rs4;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r8, %r7, 16;
-; SM70-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r9, %r8, %r6;
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r10, %r9, 16;
-; SM70-FTZ-ALLOW-NEXT: and.b32 %r11, %r10, 1;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r12, %r11, %r9;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r13, %r12, 32767;
-; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r9, %r9;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r14, %r9, 4194304;
-; SM70-FTZ-ALLOW-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r16, %rs1;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r17, %r16, 16;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r18, %rs3;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r19, %r18, 16;
-; SM70-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r20, %r19, %r17;
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r21, %r20, 16;
-; SM70-FTZ-ALLOW-NEXT: and.b32 %r22, %r21, 1;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r23, %r22, %r20;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r24, %r23, 32767;
-; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r20, %r20;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r25, %r20, 4194304;
-; SM70-FTZ-ALLOW-NEXT: selp.b32 %r26, %r25, %r24, %p2;
-; SM70-FTZ-ALLOW-NEXT: prmt.b32 %r27, %r26, %r15, 0x7632U;
-; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r27;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r28, %rs6;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r29, %r28, 16;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r98;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r30, %rs8;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r31, %r30, 16;
-; SM70-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r32, %r31, %r29;
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r33, %r32, 16;
-; SM70-FTZ-ALLOW-NEXT: and.b32 %r34, %r33, 1;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r35, %r34, %r32;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r36, %r35, 32767;
-; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p3, %r32, %r32;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r37, %r32, 4194304;
-; SM70-FTZ-ALLOW-NEXT: selp.b32 %r38, %r37, %r36, %p3;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r39, %rs5;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r40, %r39, 16;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r41, %rs7;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r42, %r41, 16;
-; SM70-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r43, %r42, %r40;
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r44, %r43, 16;
-; SM70-FTZ-ALLOW-NEXT: and.b32 %r45, %r44, 1;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r46, %r45, %r43;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r47, %r46, 32767;
-; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p4, %r43, %r43;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r48, %r43, 4194304;
-; SM70-FTZ-ALLOW-NEXT: selp.b32 %r49, %r48, %r47, %p4;
-; SM70-FTZ-ALLOW-NEXT: prmt.b32 %r50, %r49, %r38, 0x7632U;
-; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs2;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r101;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r11, %rs4;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r13, %r12, %r10;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r14, %r13, 16;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r15, %r14, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r16, %r15, %r13;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r17, %r16, 32767;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r13, %r13;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r18, %r13, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r19, %r18, %r17, %p1;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r20, %rs1;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r21, %r20, 16;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r22, %rs3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r23, %r22, 16;
+; SM70-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r24, %r23, %r21;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r25, %r24, 16;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r26, %r25, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r27, %r26, %r24;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r28, %r27, 32767;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r24, %r24;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r29, %r24, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r30, %r29, %r28, %p2;
+; SM70-FTZ-ALLOW-NEXT: prmt.b32 %r31, %r30, %r19, 0x7632U;
+; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r31;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r32, %rs6;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r33, %r32, 16;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r102;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r34, %rs8;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r36, %r35, %r33;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r37, %r36, 16;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r38, %r37, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r39, %r38, %r36;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r40, %r39, 32767;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p3, %r36, %r36;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r41, %r36, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r42, %r41, %r40, %p3;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r43, %rs5;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r44, %r43, 16;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r45, %rs7;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r46, %r45, 16;
+; SM70-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r47, %r46, %r44;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r48, %r47, 16;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r49, %r48, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r50, %r49, %r47;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r51, %r50, 32767;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p4, %r47, %r47;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r52, %r47, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r53, %r52, %r51, %p4;
+; SM70-FTZ-ALLOW-NEXT: prmt.b32 %r54, %r53, %r42, 0x7632U;
+; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r54;
; SM70-FTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r97;
-; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r98;
+; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r101;
+; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r102;
; SM70-FTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
-; SM70-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r97, %rd11;
+; SM70-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r102}, %rd11; }
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r101, %rd11;
; SM70-FTZ-ALLOW-NEXT: @%p5 bra $L__BB319_1;
; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
-; SM70-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r99, %rd12;
+; SM70-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r104}, %rd12; }
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r103, %rd12;
; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs13, %rs14}, %r4;
; SM70-FTZ-ALLOW-NEXT: $L__BB319_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r51, %rs10;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r52, %r51, 16;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r99;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r53, %rs12;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r54, %r53, 16;
-; SM70-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r55, %r54, %r52;
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r56, %r55, 16;
-; SM70-FTZ-ALLOW-NEXT: and.b32 %r57, %r56, 1;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r58, %r57, %r55;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r59, %r58, 32767;
-; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p6, %r55, %r55;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r60, %r55, 4194304;
-; SM70-FTZ-ALLOW-NEXT: selp.b32 %r61, %r60, %r59, %p6;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r62, %rs9;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r63, %r62, 16;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r64, %rs11;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r65, %r64, 16;
-; SM70-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r66, %r65, %r63;
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r67, %r66, 16;
-; SM70-FTZ-ALLOW-NEXT: and.b32 %r68, %r67, 1;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r69, %r68, %r66;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r70, %r69, 32767;
-; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p7, %r66, %r66;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r71, %r66, 4194304;
-; SM70-FTZ-ALLOW-NEXT: selp.b32 %r72, %r71, %r70, %p7;
-; SM70-FTZ-ALLOW-NEXT: prmt.b32 %r73, %r72, %r61, 0x7632U;
-; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r73;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r74, %rs14;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r75, %r74, 16;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r100;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r76, %rs16;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r77, %r76, 16;
-; SM70-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r78, %r77, %r75;
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r79, %r78, 16;
-; SM70-FTZ-ALLOW-NEXT: and.b32 %r80, %r79, 1;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r81, %r80, %r78;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r82, %r81, 32767;
-; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p8, %r78, %r78;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r83, %r78, 4194304;
-; SM70-FTZ-ALLOW-NEXT: selp.b32 %r84, %r83, %r82, %p8;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r85, %rs13;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r86, %r85, 16;
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r87, %rs15;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r88, %r87, 16;
-; SM70-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r89, %r88, %r86;
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r90, %r89, 16;
-; SM70-FTZ-ALLOW-NEXT: and.b32 %r91, %r90, 1;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r92, %r91, %r89;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r93, %r92, 32767;
-; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p9, %r89, %r89;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r94, %r89, 4194304;
-; SM70-FTZ-ALLOW-NEXT: selp.b32 %r95, %r94, %r93, %p9;
-; SM70-FTZ-ALLOW-NEXT: prmt.b32 %r96, %r95, %r84, 0x7632U;
-; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r96;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r55, %rs10;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r56, %r55, 16;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs11, %rs12}, %r103;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r57, %rs12;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r58, %r57, 16;
+; SM70-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r59, %r58, %r56;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r60, %r59, 16;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r61, %r60, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r62, %r61, %r59;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r63, %r62, 32767;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p6, %r59, %r59;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r64, %r59, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r65, %r64, %r63, %p6;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r66, %rs9;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r67, %r66, 16;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r68, %rs11;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r69, %r68, 16;
+; SM70-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r70, %r69, %r67;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r71, %r70, 16;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r72, %r71, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r73, %r72, %r70;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r74, %r73, 32767;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p7, %r70, %r70;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r75, %r70, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r76, %r75, %r74, %p7;
+; SM70-FTZ-ALLOW-NEXT: prmt.b32 %r77, %r76, %r65, 0x7632U;
+; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r77;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r78, %rs14;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r79, %r78, 16;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 {%rs15, %rs16}, %r104;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r80, %rs16;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r81, %r80, 16;
+; SM70-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r82, %r81, %r79;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r83, %r82, 16;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r84, %r83, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r85, %r84, %r82;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r86, %r85, 32767;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p8, %r82, %r82;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r87, %r82, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r88, %r87, %r86, %p8;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r89, %rs13;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r90, %r89, 16;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r91, %rs15;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r92, %r91, 16;
+; SM70-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r93, %r92, %r90;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r94, %r93, 16;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r95, %r94, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r96, %r95, %r93;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r97, %r96, 32767;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p9, %r93, %r93;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r98, %r93, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r99, %r98, %r97, %p9;
+; SM70-FTZ-ALLOW-NEXT: prmt.b32 %r100, %r99, %r88, 0x7632U;
+; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r100;
; SM70-FTZ-ALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r99;
-; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r100;
+; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r103;
+; SM70-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r104;
; SM70-FTZ-ALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
-; SM70-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
-; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r99, %rd21;
+; SM70-FTZ-ALLOW-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r104}, %rd21; }
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r103, %rd21;
; SM70-FTZ-ALLOW-NEXT: @%p10 bra $L__BB319_3;
; SM70-FTZ-ALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
+; SM70-FTZ-ALLOW-NEXT: st.param.v4.b32 [func_retval0], {%r101, %r102, %r103, %r104};
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -23684,9 +23542,9 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -23730,9 +23588,9 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -23776,9 +23634,9 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -23822,9 +23680,9 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -25080,9 +24938,9 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -25126,9 +24984,9 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -25172,9 +25030,9 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -25218,9 +25076,9 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -26476,9 +26334,9 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -26523,9 +26381,9 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -26570,9 +26428,9 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -26617,9 +26475,9 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -27876,9 +27734,9 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -27923,9 +27781,9 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -27970,9 +27828,9 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -28017,9 +27875,9 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -29276,14 +29134,14 @@ define <1 x i8> @add_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd2, [add_monotonic_v1i8_global_cta_param_0];
-; SM70-NEXT: ld.param.b8 %r6, [add_monotonic_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: and.b32 %r8, %r7, 3;
-; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
; SM70-NEXT: shl.b32 %r2, 255, %r1;
; SM70-NEXT: not.b32 %r3, %r2;
-; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.param.b8 %r8, [add_monotonic_v1i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r4, %r8, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT: $L__BB336_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
@@ -29500,14 +29358,14 @@ define <1 x i8> @add_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd2, [add_acquire_v1i8_global_cta_param_0];
-; SM70-NEXT: ld.param.b8 %r6, [add_acquire_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: and.b32 %r8, %r7, 3;
-; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
; SM70-NEXT: shl.b32 %r2, 255, %r1;
; SM70-NEXT: not.b32 %r3, %r2;
-; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.param.b8 %r8, [add_acquire_v1i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r4, %r8, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT: $L__BB340_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
@@ -29729,14 +29587,14 @@ define <1 x i8> @add_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd2, [add_release_v1i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b8 %r6, [add_release_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: and.b32 %r8, %r7, 3;
-; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
; SM70-NEXT: shl.b32 %r2, 255, %r1;
; SM70-NEXT: not.b32 %r3, %r2;
-; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.param.b8 %r8, [add_release_v1i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r4, %r8, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT: $L__BB344_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
@@ -30033,8 +29891,7 @@ define <1 x i32> @add_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v1i32_global_cta_param_0];
; SM70-NEXT: ld.param.b32 %r1, [add_acquire_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: atom.acquire.cta.global.add.u32 %r2, [%rd1], %r1;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
@@ -30113,9 +29970,8 @@ define <1 x i32> @add_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [add_release_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [add_release_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: atom.release.cta.global.add.u32 %r2, [%rd1], %r1;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
@@ -30195,14 +30051,14 @@ define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd2, [nand_monotonic_v1i8_global_cta_param_0];
-; SM70-NEXT: ld.param.b8 %r6, [nand_monotonic_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: and.b32 %r8, %r7, 3;
-; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
; SM70-NEXT: shl.b32 %r2, 255, %r1;
; SM70-NEXT: not.b32 %r3, %r2;
-; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.param.b8 %r8, [nand_monotonic_v1i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r4, %r8, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT: $L__BB360_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
@@ -30334,14 +30190,14 @@ define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd2, [nand_acquire_v1i8_global_cta_param_0];
-; SM70-NEXT: ld.param.b8 %r6, [nand_acquire_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: and.b32 %r8, %r7, 3;
-; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
; SM70-NEXT: shl.b32 %r2, 255, %r1;
; SM70-NEXT: not.b32 %r3, %r2;
-; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.param.b8 %r8, [nand_acquire_v1i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r4, %r8, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT: $L__BB364_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
@@ -30478,14 +30334,14 @@ define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd2, [nand_release_v1i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b8 %r6, [nand_release_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: and.b32 %r8, %r7, 3;
-; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
; SM70-NEXT: shl.b32 %r2, 255, %r1;
; SM70-NEXT: not.b32 %r3, %r2;
-; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.param.b8 %r8, [nand_release_v1i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r4, %r8, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT: $L__BB368_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
@@ -30619,19 +30475,19 @@ define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_monotonic_v1i32_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
-; SM70-NEXT: ld.param.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT: $L__BB372_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
; SM70-NEXT: not.b32 %r4, %r3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
; SM70-NEXT: @%p1 bra $L__BB372_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
ret <1 x i32> %retval
@@ -30849,20 +30705,20 @@ define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_acquire_v1i32_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
-; SM70-NEXT: ld.param.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT: $L__BB376_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
; SM70-NEXT: not.b32 %r4, %r3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
; SM70-NEXT: @%p1 bra $L__BB376_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
ret <1 x i32> %retval
@@ -31083,20 +30939,20 @@ define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_release_v1i32_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v1i32_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b32 %r1, [nand_release_v1i32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT: $L__BB380_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
; SM70-NEXT: not.b32 %r4, %r3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
; SM70-NEXT: @%p1 bra $L__BB380_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
ret <1 x i32> %retval
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
index 19bd7eab664888..65585e24f3c3f9 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
@@ -22,14 +22,14 @@ define <1 x i8> @xchg_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd2, [xchg_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b8 %r5, [xchg_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
-; SM90-NEXT: and.b32 %r7, %r6, 3;
-; SM90-NEXT: shl.b32 %r1, %r7, 3;
-; SM90-NEXT: shl.b32 %r8, 255, %r1;
-; SM90-NEXT: not.b32 %r2, %r8;
-; SM90-NEXT: shl.b32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: shl.b32 %r7, 255, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.param::func.b8 %r8, [xchg_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r3, %r8, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM90-NEXT: $L__BB0_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -124,8 +124,8 @@ define <1 x i16> @xchg_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd2, [xchg_acq_rel_v1i16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b16 %r5, [xchg_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
; SM90-NEXT: and.b32 %r7, %r6, 3;
@@ -210,10 +210,8 @@ define <1 x i32> @xchg_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b32 %r1, [xchg_acq_rel_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r2, [%rd1], %r1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -301,10 +299,8 @@ define <1 x i64> @xchg_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v1i64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b64 %rd2, [xchg_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd3, [%rd1], %rd2;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -417,14 +413,14 @@ define <1 x i8> @add_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd2, [add_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b8 %r6, [add_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: and.b32 %r8, %r7, 3;
-; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
; SM90-NEXT: shl.b32 %r2, 255, %r1;
; SM90-NEXT: not.b32 %r3, %r2;
-; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.param::func.b8 %r8, [add_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r4, %r8, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM90-NEXT: $L__BB16_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -820,10 +816,8 @@ define <1 x i32> @add_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b32 %r1, [add_acq_rel_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.u32 %r2, [%rd1], %r1;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -904,10 +898,8 @@ define <1 x i64> @add_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v1i64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b64 %rd2, [add_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd3, [%rd1], %rd2;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.u64 %rd3, [%rd1], %rd2;
; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -994,14 +986,14 @@ define <1 x i8> @sub_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd2, [sub_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b8 %r6, [sub_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: and.b32 %r8, %r7, 3;
-; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
; SM90-NEXT: shl.b32 %r2, 255, %r1;
; SM90-NEXT: not.b32 %r3, %r2;
-; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.param::func.b8 %r8, [sub_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r4, %r8, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM90-NEXT: $L__BB32_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -1428,11 +1420,9 @@ define <1 x i32> @sub_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b32 %r1, [sub_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: neg.s32 %r2, %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r2;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.u32 %r3, [%rd1], %r2;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -1447,14 +1437,14 @@ define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v2i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [sub_acq_rel_v2i32_global_cta_param_1];
-; SM90-NEXT: neg.s32 %r3, %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1], %r3;
-; SM90-NEXT: neg.s32 %r5, %r2;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r5;
+; SM90-NEXT: neg.s32 %r3, %r2;
+; SM90-NEXT: neg.s32 %r4, %r1;
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r4;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r4, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -1468,18 +1458,18 @@ define <4 x i32> @sub_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v4i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v4i32_global_cta_param_1];
-; SM90-NEXT: neg.s32 %r5, %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1], %r5;
+; SM90-NEXT: neg.s32 %r5, %r4;
+; SM90-NEXT: neg.s32 %r6, %r3;
; SM90-NEXT: neg.s32 %r7, %r2;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+4], %r7;
-; SM90-NEXT: neg.s32 %r9, %r3;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+8], %r9;
-; SM90-NEXT: neg.s32 %r11, %r4;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r11;
+; SM90-NEXT: neg.s32 %r8, %r1;
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r5;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -1493,28 +1483,28 @@ define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v8i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i32_global_cta_param_1];
-; SM90-NEXT: neg.s32 %r9, %r5;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1], %r9;
-; SM90-NEXT: neg.s32 %r11, %r6;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+4], %r11;
-; SM90-NEXT: neg.s32 %r13, %r7;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+8], %r13;
-; SM90-NEXT: neg.s32 %r15, %r8;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+12], %r15;
-; SM90-NEXT: neg.s32 %r17, %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r18, [%rd1+16], %r17;
-; SM90-NEXT: neg.s32 %r19, %r2;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r20, [%rd1+20], %r19;
-; SM90-NEXT: neg.s32 %r21, %r3;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r22, [%rd1+24], %r21;
-; SM90-NEXT: neg.s32 %r23, %r4;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r24, [%rd1+28], %r23;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
-; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM90-NEXT: neg.s32 %r5, %r4;
+; SM90-NEXT: neg.s32 %r6, %r3;
+; SM90-NEXT: neg.s32 %r7, %r2;
+; SM90-NEXT: neg.s32 %r8, %r1;
+; SM90-NEXT: ld.param::func.v4.b32 {%r9, %r10, %r11, %r12}, [sub_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: neg.s32 %r13, %r12;
+; SM90-NEXT: neg.s32 %r14, %r11;
+; SM90-NEXT: neg.s32 %r15, %r10;
+; SM90-NEXT: neg.s32 %r16, %r9;
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r17, [%rd1], %r16;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r18, [%rd1+4], %r15;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r19, [%rd1+8], %r14;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r20, [%rd1+12], %r13;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r21, [%rd1+16], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r22, [%rd1+20], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r23, [%rd1+24], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r24, [%rd1+28], %r5;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -1527,11 +1517,9 @@ define <1 x i64> @sub_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v1i64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b64 %rd2, [sub_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: neg.s64 %rd3, %rd2;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd3;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.u64 %rd4, [%rd1], %rd3;
; SM90-NEXT: st.param::func.b64 [func_retval0], %rd4;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -1545,14 +1533,14 @@ define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v2i64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v2i64_global_cta_param_1];
-; SM90-NEXT: neg.s64 %rd4, %rd2;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd5, [%rd1], %rd4;
-; SM90-NEXT: neg.s64 %rd6, %rd3;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd6;
+; SM90-NEXT: neg.s64 %rd4, %rd3;
+; SM90-NEXT: neg.s64 %rd5, %rd2;
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd6, [%rd1], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd5, %rd7};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd6, %rd7};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -1565,20 +1553,20 @@ define <4 x i64> @sub_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v4i64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v4i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v4i64_global_cta_param_1];
-; SM90-NEXT: neg.s64 %rd6, %rd4;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1], %rd6;
-; SM90-NEXT: neg.s64 %rd8, %rd5;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd9, [%rd1+8], %rd8;
-; SM90-NEXT: neg.s64 %rd10, %rd2;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1+16], %rd10;
-; SM90-NEXT: neg.s64 %rd12, %rd3;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd12;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd11, %rd13};
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd7, %rd9};
+; SM90-NEXT: neg.s64 %rd4, %rd3;
+; SM90-NEXT: neg.s64 %rd5, %rd2;
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: neg.s64 %rd8, %rd7;
+; SM90-NEXT: neg.s64 %rd9, %rd6;
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd10, [%rd1], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1+8], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd12, [%rd1+16], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd10, %rd11};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -1591,32 +1579,32 @@ define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v8i64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param::func.v2.b64 {%rd8, %rd9}, [sub_acq_rel_v8i64_global_cta_param_1];
-; SM90-NEXT: neg.s64 %rd10, %rd8;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1], %rd10;
-; SM90-NEXT: neg.s64 %rd12, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+8], %rd12;
-; SM90-NEXT: neg.s64 %rd14, %rd6;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd15, [%rd1+16], %rd14;
-; SM90-NEXT: neg.s64 %rd16, %rd7;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd17, [%rd1+24], %rd16;
-; SM90-NEXT: neg.s64 %rd18, %rd4;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd19, [%rd1+32], %rd18;
-; SM90-NEXT: neg.s64 %rd20, %rd5;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd21, [%rd1+40], %rd20;
-; SM90-NEXT: neg.s64 %rd22, %rd2;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd23, [%rd1+48], %rd22;
-; SM90-NEXT: neg.s64 %rd24, %rd3;
-; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd25, [%rd1+56], %rd24;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd23, %rd25};
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd19, %rd21};
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd15, %rd17};
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd11, %rd13};
+; SM90-NEXT: neg.s64 %rd4, %rd3;
+; SM90-NEXT: neg.s64 %rd5, %rd2;
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: neg.s64 %rd8, %rd7;
+; SM90-NEXT: neg.s64 %rd9, %rd6;
+; SM90-NEXT: ld.param::func.v2.b64 {%rd10, %rd11}, [sub_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: neg.s64 %rd12, %rd11;
+; SM90-NEXT: neg.s64 %rd13, %rd10;
+; SM90-NEXT: ld.param::func.v2.b64 {%rd14, %rd15}, [sub_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: neg.s64 %rd16, %rd15;
+; SM90-NEXT: neg.s64 %rd17, %rd14;
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd18, [%rd1], %rd17;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd19, [%rd1+8], %rd16;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd20, [%rd1+16], %rd13;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd21, [%rd1+24], %rd12;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd22, [%rd1+32], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd23, [%rd1+40], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd24, [%rd1+48], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd25, [%rd1+56], %rd4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd24, %rd25};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd22, %rd23};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd20, %rd21};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd18, %rd19};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -1631,17 +1619,17 @@ define <1 x i8> @and_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b8 %r1, [and_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd2, %rd1, -4;
-; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
-; SM90-NEXT: and.b32 %r3, %r2, 3;
-; SM90-NEXT: shl.b32 %r4, %r3, 3;
-; SM90-NEXT: shl.b32 %r5, 255, %r4;
-; SM90-NEXT: not.b32 %r6, %r5;
-; SM90-NEXT: shl.b32 %r7, %r1, %r4;
-; SM90-NEXT: or.b32 %r8, %r7, %r6;
+; SM90-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM90-NEXT: and.b32 %r2, %r1, 3;
+; SM90-NEXT: shl.b32 %r3, %r2, 3;
+; SM90-NEXT: shl.b32 %r4, 255, %r3;
+; SM90-NEXT: not.b32 %r5, %r4;
+; SM90-NEXT: ld.param::func.b8 %r6, [and_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r7, %r6, %r3;
+; SM90-NEXT: or.b32 %r8, %r7, %r5;
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r9, [%rd2], %r8;
-; SM90-NEXT: shr.u32 %r10, %r9, %r4;
+; SM90-NEXT: shr.u32 %r10, %r9, %r3;
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param::func.b8 [func_retval0], %r10;
; SM90-NEXT: ret;
@@ -1912,10 +1900,8 @@ define <1 x i32> @and_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b32 %r1, [and_acq_rel_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r2, [%rd1], %r1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.and.b32 %r2, [%rd1], %r1;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -1996,10 +1982,8 @@ define <1 x i64> @and_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v1i64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b64 %rd2, [and_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd3, [%rd1], %rd2;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.and.b64 %rd3, [%rd1], %rd2;
; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -2086,14 +2070,14 @@ define <1 x i8> @nand_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd2, [nand_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b8 %r6, [nand_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: and.b32 %r8, %r7, 3;
-; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
; SM90-NEXT: shl.b32 %r2, 255, %r1;
; SM90-NEXT: not.b32 %r3, %r2;
-; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.param::func.b8 %r8, [nand_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r4, %r8, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM90-NEXT: $L__BB64_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -2412,21 +2396,21 @@ define <1 x i32> @nand_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b32 %r2, [nand_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b32 %r1, [nand_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB72_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
; SM90-NEXT: not.b32 %r4, %r3;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
; SM90-NEXT: @%p1 bra $L__BB72_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -2676,21 +2660,21 @@ define <1 x i64> @nand_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM90-NEXT: .reg .b64 %rd<7>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [nand_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd2];
; SM90-NEXT: $L__BB76_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: and.b64 %rd4, %rd6, %rd1;
+; SM90-NEXT: and.b64 %rd4, %rd6, %rd3;
; SM90-NEXT: not.b64 %rd5, %rd4;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
-; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
-; SM90-NEXT: mov.b64 %rd6, %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5;
+; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd6;
+; SM90-NEXT: mov.b64 %rd6, %rd1;
; SM90-NEXT: @%p1 bra $L__BB76_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -2967,14 +2951,14 @@ define <1 x i8> @or_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %va
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b8 %r1, [or_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd2, %rd1, -4;
-; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
-; SM90-NEXT: and.b32 %r3, %r2, 3;
-; SM90-NEXT: shl.b32 %r4, %r3, 3;
-; SM90-NEXT: shl.b32 %r5, %r1, %r4;
+; SM90-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM90-NEXT: and.b32 %r2, %r1, 3;
+; SM90-NEXT: shl.b32 %r3, %r2, 3;
+; SM90-NEXT: ld.param::func.b8 %r4, [or_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r5, %r4, %r3;
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
-; SM90-NEXT: shr.u32 %r7, %r6, %r4;
+; SM90-NEXT: shr.u32 %r7, %r6, %r3;
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param::func.b8 [func_retval0], %r7;
; SM90-NEXT: ret;
@@ -3239,10 +3223,8 @@ define <1 x i32> @or_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b32 %r1, [or_acq_rel_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r2, [%rd1], %r1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.or.b32 %r2, [%rd1], %r1;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -3323,10 +3305,8 @@ define <1 x i64> @or_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v1i64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b64 %rd2, [or_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd3, [%rd1], %rd2;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.or.b64 %rd3, [%rd1], %rd2;
; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -3412,14 +3392,14 @@ define <1 x i8> @xor_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b8 %r1, [xor_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd2, %rd1, -4;
-; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
-; SM90-NEXT: and.b32 %r3, %r2, 3;
-; SM90-NEXT: shl.b32 %r4, %r3, 3;
-; SM90-NEXT: shl.b32 %r5, %r1, %r4;
+; SM90-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM90-NEXT: and.b32 %r2, %r1, 3;
+; SM90-NEXT: shl.b32 %r3, %r2, 3;
+; SM90-NEXT: ld.param::func.b8 %r4, [xor_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r5, %r4, %r3;
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
-; SM90-NEXT: shr.u32 %r7, %r6, %r4;
+; SM90-NEXT: shr.u32 %r7, %r6, %r3;
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param::func.b8 [func_retval0], %r7;
; SM90-NEXT: ret;
@@ -3684,10 +3664,8 @@ define <1 x i32> @xor_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b32 %r1, [xor_acq_rel_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r2, [%rd1], %r1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.xor.b32 %r2, [%rd1], %r1;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -3768,10 +3746,8 @@ define <1 x i64> @xor_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v1i64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b64 %rd2, [xor_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd3, [%rd1], %rd2;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.xor.b64 %rd3, [%rd1], %rd2;
; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -3852,14 +3828,14 @@ define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-LABEL: max_acq_rel_v1i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b16 %rs<5>;
; SM90-NEXT: .reg .b32 %r<15>;
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [max_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.s8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -3867,12 +3843,13 @@ define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: shl.b32 %r6, 255, %r1;
; SM90-NEXT: not.b32 %r2, %r6;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: cvt.s16.s8 %rs3, %rs1;
; SM90-NEXT: $L__BB112_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r7, %r14, %r1;
; SM90-NEXT: cvt.s8.s32 %rs2, %r7;
-; SM90-NEXT: max.s16 %rs3, %rs2, %rs1;
-; SM90-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM90-NEXT: max.s16 %rs4, %rs2, %rs3;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs4;
; SM90-NEXT: and.b32 %r9, %r8, 255;
; SM90-NEXT: shl.b32 %r10, %r9, %r1;
; SM90-NEXT: and.b32 %r11, %r14, %r2;
@@ -4121,9 +4098,9 @@ define <1 x i16> @max_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [max_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -4296,10 +4273,8 @@ define <1 x i32> @max_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b32 %r1, [max_acq_rel_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r2, [%rd1], %r1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.max.s32 %r2, [%rd1], %r1;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -4380,10 +4355,8 @@ define <1 x i64> @max_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v1i64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b64 %rd2, [max_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd3, [%rd1], %rd2;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.max.s64 %rd3, [%rd1], %rd2;
; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -4464,14 +4437,14 @@ define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-LABEL: min_acq_rel_v1i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b16 %rs<5>;
; SM90-NEXT: .reg .b32 %r<15>;
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [min_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.s8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -4479,12 +4452,13 @@ define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: shl.b32 %r6, 255, %r1;
; SM90-NEXT: not.b32 %r2, %r6;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: cvt.s16.s8 %rs3, %rs1;
; SM90-NEXT: $L__BB128_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r7, %r14, %r1;
; SM90-NEXT: cvt.s8.s32 %rs2, %r7;
-; SM90-NEXT: min.s16 %rs3, %rs2, %rs1;
-; SM90-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM90-NEXT: min.s16 %rs4, %rs2, %rs3;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs4;
; SM90-NEXT: and.b32 %r9, %r8, 255;
; SM90-NEXT: shl.b32 %r10, %r9, %r1;
; SM90-NEXT: and.b32 %r11, %r14, %r2;
@@ -4733,9 +4707,9 @@ define <1 x i16> @min_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [min_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -4908,10 +4882,8 @@ define <1 x i32> @min_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b32 %r1, [min_acq_rel_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r2, [%rd1], %r1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.min.s32 %r2, [%rd1], %r1;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -4992,10 +4964,8 @@ define <1 x i64> @min_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v1i64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b64 %rd2, [min_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd3, [%rd1], %rd2;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.min.s64 %rd3, [%rd1], %rd2;
; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -5316,9 +5286,9 @@ define <1 x i16> @umax_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [umax_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -5491,10 +5461,8 @@ define <1 x i32> @umax_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b32 %r1, [umax_acq_rel_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r2, [%rd1], %r1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.max.u32 %r2, [%rd1], %r1;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -5575,10 +5543,8 @@ define <1 x i64> @umax_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v1i64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b64 %rd2, [umax_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd3, [%rd1], %rd2;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.max.u64 %rd3, [%rd1], %rd2;
; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -5899,9 +5865,9 @@ define <1 x i16> @umin_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [umin_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -6074,10 +6040,8 @@ define <1 x i32> @umin_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b32 %r1, [umin_acq_rel_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r2, [%rd1], %r1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.min.u32 %r2, [%rd1], %r1;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -6158,10 +6122,8 @@ define <1 x i64> @umin_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v1i64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b64 %rd2, [umin_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd3, [%rd1], %rd2;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.min.u64 %rd3, [%rd1], %rd2;
; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
@@ -6505,9 +6467,9 @@ define <1 x i16> @uinc_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [uinc_wrap_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -6744,10 +6706,8 @@ define <1 x i32> @uinc_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [uinc_wrap_acq_rel_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b32 %r1, [uinc_wrap_acq_rel_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r2, [%rd1], %r1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.inc.u32 %r2, [%rd1], %r1;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -6828,22 +6788,22 @@ define <1 x i64> @uinc_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<7>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b64 %rd1, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd2];
; SM90-NEXT: $L__BB188_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s64 %rd4, %rd6, 1;
-; SM90-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM90-NEXT: setp.ge.u64 %p1, %rd6, %rd3;
; SM90-NEXT: selp.b64 %rd5, 0, %rd4, %p1;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
-; SM90-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
-; SM90-NEXT: mov.b64 %rd6, %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5;
+; SM90-NEXT: setp.ne.b64 %p2, %rd1, %rd6;
+; SM90-NEXT: mov.b64 %rd6, %rd1;
; SM90-NEXT: @%p2 bra $L__BB188_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -7428,9 +7388,9 @@ define <1 x i16> @udec_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [udec_wrap_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -7694,10 +7654,8 @@ define <1 x i32> @udec_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b32 %r1, [udec_wrap_acq_rel_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r2, [%rd1], %r1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.dec.u32 %r2, [%rd1], %r1;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
@@ -7778,24 +7736,24 @@ define <1 x i64> @udec_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<8>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd7, [%rd3];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd7, [%rd2];
; SM90-NEXT: $L__BB204_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s64 %rd4, %rd7, -1;
; SM90-NEXT: setp.eq.b64 %p1, %rd7, 0;
-; SM90-NEXT: setp.gt.u64 %p2, %rd7, %rd1;
-; SM90-NEXT: selp.b64 %rd5, %rd1, %rd4, %p2;
-; SM90-NEXT: selp.b64 %rd6, %rd1, %rd5, %p1;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd7, %rd6;
-; SM90-NEXT: setp.ne.b64 %p3, %rd2, %rd7;
-; SM90-NEXT: mov.b64 %rd7, %rd2;
+; SM90-NEXT: setp.gt.u64 %p2, %rd7, %rd3;
+; SM90-NEXT: selp.b64 %rd5, %rd3, %rd4, %p2;
+; SM90-NEXT: selp.b64 %rd6, %rd3, %rd5, %p1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd7, %rd6;
+; SM90-NEXT: setp.ne.b64 %p3, %rd1, %rd7;
+; SM90-NEXT: mov.b64 %rd7, %rd1;
; SM90-NEXT: @%p3 bra $L__BB204_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -8382,9 +8340,9 @@ define <1 x i16> @usub_cond_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [usub_cond_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -8618,22 +8576,22 @@ define <1 x i32> @usub_cond_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b32 %r2, [usub_cond_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [usub_cond_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b32 %r1, [usub_cond_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB216_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.ge.u32 %p1, %r5, %r1;
-; SM90-NEXT: sub.s32 %r3, %r5, %r1;
+; SM90-NEXT: setp.ge.u32 %p1, %r5, %r2;
+; SM90-NEXT: sub.s32 %r3, %r5, %r2;
; SM90-NEXT: selp.b32 %r4, %r3, %r5, %p1;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM90-NEXT: setp.ne.b32 %p2, %r2, %r5;
-; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p2, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
; SM90-NEXT: @%p2 bra $L__BB216_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -8904,22 +8862,22 @@ define <1 x i64> @usub_cond_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<7>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [usub_cond_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b64 %rd1, [usub_cond_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd2];
; SM90-NEXT: $L__BB220_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
-; SM90-NEXT: sub.s64 %rd4, %rd6, %rd1;
+; SM90-NEXT: setp.ge.u64 %p1, %rd6, %rd3;
+; SM90-NEXT: sub.s64 %rd4, %rd6, %rd3;
; SM90-NEXT: selp.b64 %rd5, %rd4, %rd6, %p1;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
-; SM90-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
-; SM90-NEXT: mov.b64 %rd6, %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5;
+; SM90-NEXT: setp.ne.b64 %p2, %rd1, %rd6;
+; SM90-NEXT: mov.b64 %rd6, %rd1;
; SM90-NEXT: @%p2 bra $L__BB220_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -9462,9 +9420,9 @@ define <1 x i16> @usub_sat_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [usub_sat_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -9676,21 +9634,21 @@ define <1 x i32> @usub_sat_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b32 %r2, [usub_sat_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [usub_sat_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b32 %r1, [usub_sat_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB232_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: max.u32 %r3, %r5, %r1;
-; SM90-NEXT: sub.s32 %r4, %r3, %r1;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: max.u32 %r3, %r5, %r2;
+; SM90-NEXT: sub.s32 %r4, %r3, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
; SM90-NEXT: @%p1 bra $L__BB232_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -9947,21 +9905,21 @@ define <1 x i64> @usub_sat_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<7>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [usub_sat_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b64 %rd1, [usub_sat_acq_rel_v1i64_global_cta_param_1];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd2];
; SM90-NEXT: $L__BB236_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: max.u64 %rd4, %rd6, %rd1;
-; SM90-NEXT: sub.s64 %rd5, %rd4, %rd1;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
-; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
-; SM90-NEXT: mov.b64 %rd6, %rd2;
+; SM90-NEXT: max.u64 %rd4, %rd6, %rd3;
+; SM90-NEXT: sub.s64 %rd5, %rd4, %rd3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5;
+; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd6;
+; SM90-NEXT: mov.b64 %rd6, %rd1;
; SM90-NEXT: @%p1 bra $L__BB236_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -10237,20 +10195,20 @@ define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fadd_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NOFTZ-DISALLOW-NEXT: $L__BB240_1: // %atomicrmw.start
; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB240_1;
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fadd_acq_rel_v1f32_global_cta(
@@ -10260,10 +10218,8 @@ define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -10274,10 +10230,8 @@ define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-FTZ-DISALLOW-NEXT: ret;
;
@@ -10288,10 +10242,8 @@ define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
@@ -10341,11 +10293,8 @@ define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v2.f32 {%r3, %r4}, [%rd1], {%r1, %r2};
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -10356,11 +10305,8 @@ define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.v2.f32 {%r3, %r4}, [%rd1], {%r1, %r2};
; SM90-FTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-FTZ-DISALLOW-NEXT: ret;
;
@@ -10371,11 +10317,8 @@ define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v2.f32 {%r3, %r4}, [%rd1], {%r1, %r2};
; SM90-FTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
@@ -10455,13 +10398,8 @@ define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v4.f32 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -10472,13 +10410,8 @@ define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.v4.f32 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-FTZ-DISALLOW-NEXT: ret;
;
@@ -10489,13 +10422,8 @@ define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v4.f32 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
@@ -10631,14 +10559,8 @@ define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r5, %r6, %r7, %r8};
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r13, %r14, %r15, %r16}, [%rd1+16], {%r1, %r2, %r3, %r4};
; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -10654,14 +10576,8 @@ define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r5, %r6, %r7, %r8};
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r13, %r14, %r15, %r16}, [%rd1+16], {%r1, %r2, %r3, %r4};
; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -10677,14 +10593,8 @@ define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r5, %r6, %r7, %r8};
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r13, %r14, %r15, %r16}, [%rd1+16], {%r1, %r2, %r3, %r4};
; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -10701,20 +10611,20 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NOFTZ-DISALLOW-NEXT: $L__BB244_1: // %atomicrmw.start
; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r3, %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB244_1;
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v1f32_global_cta(
@@ -10724,11 +10634,9 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r2, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r2;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r3, [%rd1], %r2;
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -10739,11 +10647,9 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r2, %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r2;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r3, [%rd1], %r2;
; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-FTZ-DISALLOW-NEXT: ret;
;
@@ -10754,11 +10660,9 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r2, %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r2;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r3, [%rd1], %r2;
; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
@@ -10808,14 +10712,11 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r3, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1], %r3;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r5, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r5;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r4, %r6};
+; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r3, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r4, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v2.f32 {%r5, %r6}, [%rd1], {%r4, %r3};
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
; SM90-FTZ-DISALLOW-LABEL: fsub_acq_rel_v2f32_global_cta(
@@ -10825,14 +10726,11 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r3, %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1], %r3;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r5, %r2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r5;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r4, %r6};
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r3, %r2;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r4, %r1;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.v2.f32 {%r5, %r6}, [%rd1], {%r4, %r3};
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-FTZ-DISALLOW-NEXT: ret;
;
; SM90-FTZ-ALLOW-LABEL: fsub_acq_rel_v2f32_global_cta(
@@ -10842,14 +10740,11 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r3, %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1], %r3;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r5, %r2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r5;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r4, %r6};
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r3, %r2;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r4, %r1;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v2.f32 {%r5, %r6}, [%rd1], {%r4, %r3};
+; SM90-FTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
@@ -10928,18 +10823,13 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r5, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1], %r5;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r5, %r4;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r6, %r3;
; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r7, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+4], %r7;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r9, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+8], %r9;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r11, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r11;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r8, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
; SM90-FTZ-DISALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
@@ -10949,18 +10839,13 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r5, %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1], %r5;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r5, %r4;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r6, %r3;
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r7, %r2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+4], %r7;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r9, %r3;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+8], %r9;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r11, %r4;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r11;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r8, %r1;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-FTZ-DISALLOW-NEXT: ret;
;
; SM90-FTZ-ALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
@@ -10970,18 +10855,13 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r5, %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1], %r5;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r5, %r4;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r6, %r3;
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r7, %r2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+4], %r7;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r9, %r3;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+8], %r9;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r11, %r4;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r11;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r8, %r1;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
+; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
@@ -10991,14 +10871,22 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
; SM90-NOFTZ-DISALLOW: {
; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<25>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<33>;
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<50>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r9, %r10, %r11, %r12}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f32 %r4, %r12;
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f32 %r3, %r11;
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f32 %r2, %r10;
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f32 %r1, %r9;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r13, %r14, %r15, %r16}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f32 %r8, %r16;
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f32 %r7, %r15;
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f32 %r6, %r14;
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f32 %r5, %r13;
; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 %rd2, 0;
; SM90-NOFTZ-DISALLOW-NEXT: {
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
@@ -11007,30 +10895,30 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 {%rd3, %rd4}, dst;
; SM90-NOFTZ-DISALLOW-NEXT: }
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r20}, %rd4;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r19, %rd4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r18}, %rd3;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r28}, %rd4;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r27, %rd4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r26}, %rd3;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r25, %rd3;
; SM90-NOFTZ-DISALLOW-NEXT: $L__BB247_1: // %atomicrmw.start
; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r9, %r17, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd5, %r9;
-; SM90-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r10, %r18, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r17, %r25, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd5, %r17;
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r18, %r26, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd6, %r18;
; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd7, %rd6, 32;
; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r11, %r19, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd9, %r11;
-; SM90-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r12, %r20, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r19, %r27, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd9, %r19;
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r20, %r28, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd10, %r20;
; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd11, %rd10, 32;
; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r17;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r18;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r26;
; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r19;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r28;
; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM90-NOFTZ-DISALLOW-NEXT: {
@@ -11043,10 +10931,10 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NOFTZ-DISALLOW-NEXT: xor.b64 %rd23, %rd22, %rd20;
; SM90-NOFTZ-DISALLOW-NEXT: xor.b64 %rd24, %rd21, %rd16;
; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd25, %rd24, %rd23;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r20}, %rd22;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r19, %rd22;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r18}, %rd21;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r17, %rd21;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r28}, %rd22;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r27, %rd22;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r26}, %rd21;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r25, %rd21;
; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd25, 0;
; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB247_1;
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
@@ -11058,30 +10946,30 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 {%rd27, %rd28}, dst;
; SM90-NOFTZ-DISALLOW-NEXT: }
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r24}, %rd28;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r23, %rd28;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r22}, %rd27;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r21, %rd27;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r32}, %rd28;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r31, %rd28;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r30}, %rd27;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r29, %rd27;
; SM90-NOFTZ-DISALLOW-NEXT: $L__BB247_3: // %atomicrmw.start2
; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r13, %r21, %r5;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd29, %r13;
-; SM90-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r14, %r22, %r6;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd30, %r14;
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r21, %r29, %r5;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd29, %r21;
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r22, %r30, %r6;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd30, %r22;
; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd31, %rd30, 32;
; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd32, %rd29, %rd31;
-; SM90-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r15, %r23, %r7;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd33, %r15;
-; SM90-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r16, %r24, %r8;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r23, %r31, %r7;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd33, %r23;
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r24, %r32, %r8;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd34, %r24;
; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd35, %rd34, 32;
; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd36, %rd33, %rd35;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd37, %r21;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd38, %r22;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd37, %r29;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd38, %r30;
; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd39, %rd38, 32;
; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd40, %rd37, %rd39;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd41, %r23;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd42, %r24;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd41, %r31;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd42, %r32;
; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd43, %rd42, 32;
; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd44, %rd41, %rd43;
; SM90-NOFTZ-DISALLOW-NEXT: {
@@ -11094,16 +10982,16 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NOFTZ-DISALLOW-NEXT: xor.b64 %rd47, %rd46, %rd44;
; SM90-NOFTZ-DISALLOW-NEXT: xor.b64 %rd48, %rd45, %rd40;
; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd49, %rd48, %rd47;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r24}, %rd46;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r23, %rd46;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r22}, %rd45;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r21, %rd45;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r32}, %rd46;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r31, %rd46;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r30}, %rd45;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r29, %rd45;
; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p2, %rd49, 0;
; SM90-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB247_3;
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
@@ -11113,28 +11001,22 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r9, %r5;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1], %r9;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r11, %r6;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+4], %r11;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r13, %r7;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+8], %r13;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r15, %r8;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+12], %r15;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r17, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r18, [%rd1+16], %r17;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r19, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r20, [%rd1+20], %r19;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r21, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r22, [%rd1+24], %r21;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r23, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r23;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r5, %r4;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r6, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r7, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r8, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r9, %r10, %r11, %r12}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r13, %r12;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r14, %r11;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r15, %r10;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f32 %r16, %r9;
+; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r17, %r18, %r19, %r20}, [%rd1], {%r16, %r15, %r14, %r13};
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r21, %r22, %r23, %r24}, [%rd1+16], {%r8, %r7, %r6, %r5};
; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
; SM90-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
@@ -11144,28 +11026,22 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r9, %r5;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1], %r9;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r11, %r6;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+4], %r11;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r13, %r7;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+8], %r13;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r15, %r8;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+12], %r15;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r17, %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r18, [%rd1+16], %r17;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r19, %r2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r20, [%rd1+20], %r19;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r21, %r3;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r22, [%rd1+24], %r21;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r23, %r4;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r23;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r5, %r4;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r6, %r3;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r7, %r2;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r8, %r1;
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r9, %r10, %r11, %r12}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r13, %r12;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r14, %r11;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r15, %r10;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f32 %r16, %r9;
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r17, %r18, %r19, %r20}, [%rd1], {%r16, %r15, %r14, %r13};
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r21, %r22, %r23, %r24}, [%rd1+16], {%r8, %r7, %r6, %r5};
; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-FTZ-DISALLOW-NEXT: ret;
;
; SM90-FTZ-ALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
@@ -11175,28 +11051,22 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r9, %r5;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1], %r9;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r11, %r6;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+4], %r11;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r13, %r7;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+8], %r13;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r15, %r8;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+12], %r15;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r17, %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r18, [%rd1+16], %r17;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r19, %r2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r20, [%rd1+20], %r19;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r21, %r3;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r22, [%rd1+24], %r21;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r23, %r4;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r23;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r5, %r4;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r6, %r3;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r7, %r2;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r8, %r1;
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r9, %r10, %r11, %r12}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r13, %r12;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r14, %r11;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r15, %r10;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f32 %r16, %r9;
+; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r17, %r18, %r19, %r20}, [%rd1], {%r16, %r15, %r14, %r13};
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.v4.f32 {%r21, %r22, %r23, %r24}, [%rd1+16], {%r8, %r7, %r6, %r5};
; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
@@ -11210,20 +11080,20 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NOFTZ-DISALLOW-NEXT: $L__BB248_1: // %atomicrmw.start
; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: min.f32 %r3, %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: min.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB248_1;
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v1f32_global_cta(
@@ -11233,20 +11103,20 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NOFTZ-ALLOW-NEXT: $L__BB248_1: // %atomicrmw.start
; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: min.f32 %r3, %r4, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: min.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB248_1;
; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
; SM90-FTZ-DISALLOW-LABEL: fmin_acq_rel_v1f32_global_cta(
@@ -11256,20 +11126,20 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-FTZ-DISALLOW-NEXT: $L__BB248_1: // %atomicrmw.start
; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f32 %r3, %r4, %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB248_1;
; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-FTZ-DISALLOW-NEXT: ret;
;
; SM90-FTZ-ALLOW-LABEL: fmin_acq_rel_v1f32_global_cta(
@@ -11279,20 +11149,20 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-FTZ-ALLOW-NEXT: $L__BB248_1: // %atomicrmw.start
; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f32 %r3, %r4, %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-FTZ-ALLOW-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB248_1;
; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -12190,20 +12060,20 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NOFTZ-DISALLOW-NEXT: $L__BB252_1: // %atomicrmw.start
; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: max.f32 %r3, %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: max.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB252_1;
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v1f32_global_cta(
@@ -12213,20 +12083,20 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NOFTZ-ALLOW-NEXT: $L__BB252_1: // %atomicrmw.start
; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: max.f32 %r3, %r4, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: max.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB252_1;
; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
; SM90-FTZ-DISALLOW-LABEL: fmax_acq_rel_v1f32_global_cta(
@@ -12236,20 +12106,20 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-FTZ-DISALLOW-NEXT: $L__BB252_1: // %atomicrmw.start
; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f32 %r3, %r4, %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB252_1;
; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-FTZ-DISALLOW-NEXT: ret;
;
; SM90-FTZ-ALLOW-LABEL: fmax_acq_rel_v1f32_global_cta(
@@ -12259,20 +12129,20 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-FTZ-ALLOW-NEXT: $L__BB252_1: // %atomicrmw.start
; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f32 %r3, %r4, %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-FTZ-ALLOW-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB252_1;
; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -13170,20 +13040,20 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NOFTZ-DISALLOW-NEXT: $L__BB256_1: // %atomicrmw.start
; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: min.NaN.f32 %r3, %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: min.NaN.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB256_1;
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v1f32_global_cta(
@@ -13193,20 +13063,20 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NOFTZ-ALLOW-NEXT: $L__BB256_1: // %atomicrmw.start
; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: min.NaN.f32 %r3, %r4, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: min.NaN.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB256_1;
; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
; SM90-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v1f32_global_cta(
@@ -13216,20 +13086,20 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-FTZ-DISALLOW-NEXT: $L__BB256_1: // %atomicrmw.start
; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.NaN.f32 %r3, %r4, %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: min.ftz.NaN.f32 %r3, %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB256_1;
; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-FTZ-DISALLOW-NEXT: ret;
;
; SM90-FTZ-ALLOW-LABEL: fminimum_acq_rel_v1f32_global_cta(
@@ -13239,20 +13109,20 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-FTZ-ALLOW-NEXT: $L__BB256_1: // %atomicrmw.start
; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: min.ftz.NaN.f32 %r3, %r4, %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-FTZ-ALLOW-NEXT: min.ftz.NaN.f32 %r3, %r4, %r2;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB256_1;
; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -14150,20 +14020,20 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NOFTZ-DISALLOW-NEXT: $L__BB260_1: // %atomicrmw.start
; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: max.NaN.f32 %r3, %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: max.NaN.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB260_1;
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v1f32_global_cta(
@@ -14173,20 +14043,20 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NOFTZ-ALLOW-NEXT: $L__BB260_1: // %atomicrmw.start
; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: max.NaN.f32 %r3, %r4, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: max.NaN.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB260_1;
; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
; SM90-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v1f32_global_cta(
@@ -14196,20 +14066,20 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-FTZ-DISALLOW-NEXT: $L__BB260_1: // %atomicrmw.start
; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.NaN.f32 %r3, %r4, %r1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: max.ftz.NaN.f32 %r3, %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB260_1;
; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-FTZ-DISALLOW-NEXT: ret;
;
; SM90-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v1f32_global_cta(
@@ -14219,20 +14089,20 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-FTZ-ALLOW-NEXT: $L__BB260_1: // %atomicrmw.start
; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: max.ftz.NaN.f32 %r3, %r4, %r1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r2, %r4;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r2;
+; SM90-FTZ-ALLOW-NEXT: max.ftz.NaN.f32 %r3, %r4, %r2;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB260_1;
; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -15129,10 +14999,8 @@ define <1 x double> @fadd_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v1f64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b64 %rd2, [fadd_acq_rel_v1f64_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd3, [%rd1], %rd2;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.f64 %rd3, [%rd1], %rd2;
; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
@@ -15216,11 +15084,9 @@ define <1 x double> @fsub_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v1f64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b64 %rd2, [fsub_acq_rel_v1f64_global_cta_param_1];
; SM90-NEXT: neg.f64 %rd3, %rd2;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd4, [%rd1], %rd3;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.f64 %rd4, [%rd1], %rd3;
; SM90-NEXT: st.param::func.b64 [func_retval0], %rd4;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
@@ -15234,14 +15100,14 @@ define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [fsub_acq_rel_v2f64_global_cta_param_1];
-; SM90-NEXT: neg.f64 %rd4, %rd2;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd5, [%rd1], %rd4;
-; SM90-NEXT: neg.f64 %rd6, %rd3;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd7, [%rd1+8], %rd6;
+; SM90-NEXT: neg.f64 %rd4, %rd3;
+; SM90-NEXT: neg.f64 %rd5, %rd2;
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd6, [%rd1], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd7, [%rd1+8], %rd4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd5, %rd7};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd6, %rd7};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -15254,20 +15120,20 @@ define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v4f64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
-; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [fsub_acq_rel_v4f64_global_cta_param_1];
-; SM90-NEXT: neg.f64 %rd6, %rd4;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd7, [%rd1], %rd6;
-; SM90-NEXT: neg.f64 %rd8, %rd5;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd9, [%rd1+8], %rd8;
-; SM90-NEXT: neg.f64 %rd10, %rd2;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd11, [%rd1+16], %rd10;
-; SM90-NEXT: neg.f64 %rd12, %rd3;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd13, [%rd1+24], %rd12;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd11, %rd13};
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd7, %rd9};
+; SM90-NEXT: neg.f64 %rd4, %rd3;
+; SM90-NEXT: neg.f64 %rd5, %rd2;
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [fsub_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: neg.f64 %rd8, %rd7;
+; SM90-NEXT: neg.f64 %rd9, %rd6;
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd10, [%rd1], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd11, [%rd1+8], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd12, [%rd1+16], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd13, [%rd1+24], %rd4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd10, %rd11};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
ret <4 x double> %retval
@@ -15280,32 +15146,32 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v8f64_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [fsub_acq_rel_v8f64_global_cta_param_1+48];
-; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
-; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
-; SM90-NEXT: ld.param::func.v2.b64 {%rd8, %rd9}, [fsub_acq_rel_v8f64_global_cta_param_1];
-; SM90-NEXT: neg.f64 %rd10, %rd8;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd11, [%rd1], %rd10;
-; SM90-NEXT: neg.f64 %rd12, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd13, [%rd1+8], %rd12;
-; SM90-NEXT: neg.f64 %rd14, %rd6;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd15, [%rd1+16], %rd14;
-; SM90-NEXT: neg.f64 %rd16, %rd7;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd17, [%rd1+24], %rd16;
-; SM90-NEXT: neg.f64 %rd18, %rd4;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd19, [%rd1+32], %rd18;
-; SM90-NEXT: neg.f64 %rd20, %rd5;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd21, [%rd1+40], %rd20;
-; SM90-NEXT: neg.f64 %rd22, %rd2;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd23, [%rd1+48], %rd22;
-; SM90-NEXT: neg.f64 %rd24, %rd3;
-; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd25, [%rd1+56], %rd24;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd23, %rd25};
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd19, %rd21};
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd15, %rd17};
-; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd11, %rd13};
+; SM90-NEXT: neg.f64 %rd4, %rd3;
+; SM90-NEXT: neg.f64 %rd5, %rd2;
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: neg.f64 %rd8, %rd7;
+; SM90-NEXT: neg.f64 %rd9, %rd6;
+; SM90-NEXT: ld.param::func.v2.b64 {%rd10, %rd11}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: neg.f64 %rd12, %rd11;
+; SM90-NEXT: neg.f64 %rd13, %rd10;
+; SM90-NEXT: ld.param::func.v2.b64 {%rd14, %rd15}, [fsub_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: neg.f64 %rd16, %rd15;
+; SM90-NEXT: neg.f64 %rd17, %rd14;
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd18, [%rd1], %rd17;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd19, [%rd1+8], %rd16;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd20, [%rd1+16], %rd13;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd21, [%rd1+24], %rd12;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd22, [%rd1+32], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd23, [%rd1+40], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd24, [%rd1+48], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd25, [%rd1+56], %rd4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd24, %rd25};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd22, %rd23};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd20, %rd21};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd18, %rd19};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -15318,20 +15184,20 @@ define <1 x double> @fmin_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fmin_acq_rel_v1f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v1f64_global_cta_param_1];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd2];
; SM90-NEXT: $L__BB272_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: min.f64 %rd4, %rd5, %rd1;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
-; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
-; SM90-NEXT: mov.b64 %rd5, %rd2;
+; SM90-NEXT: min.f64 %rd4, %rd5, %rd3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd5, %rd4;
+; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd5;
+; SM90-NEXT: mov.b64 %rd5, %rd1;
; SM90-NEXT: @%p1 bra $L__BB272_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -15592,20 +15458,20 @@ define <1 x double> @fmax_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v1f64_global_cta_param_1];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd2];
; SM90-NEXT: $L__BB276_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: max.f64 %rd4, %rd5, %rd1;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
-; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
-; SM90-NEXT: mov.b64 %rd5, %rd2;
+; SM90-NEXT: max.f64 %rd4, %rd5, %rd3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd5, %rd4;
+; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd5;
+; SM90-NEXT: mov.b64 %rd5, %rd1;
; SM90-NEXT: @%p1 bra $L__BB276_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -15866,26 +15732,26 @@ define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: .reg .b64 %rd<9>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fminimum_acq_rel_v1f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd3];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd2];
; SM90-NEXT: $L__BB280_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.eq.b64 %p1, %rd8, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd4, %rd8, %rd1, %p1;
-; SM90-NEXT: setp.nan.f64 %p2, %rd8, %rd1;
-; SM90-NEXT: min.f64 %rd5, %rd8, %rd1;
+; SM90-NEXT: selp.f64 %rd4, %rd8, %rd3, %p1;
+; SM90-NEXT: setp.nan.f64 %p2, %rd8, %rd3;
+; SM90-NEXT: min.f64 %rd5, %rd8, %rd3;
; SM90-NEXT: selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
-; SM90-NEXT: setp.eq.f64 %p3, %rd8, %rd1;
+; SM90-NEXT: setp.eq.f64 %p3, %rd8, %rd3;
; SM90-NEXT: selp.f64 %rd7, %rd4, %rd6, %p3;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
-; SM90-NEXT: setp.ne.b64 %p4, %rd2, %rd8;
-; SM90-NEXT: mov.b64 %rd8, %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd8, %rd7;
+; SM90-NEXT: setp.ne.b64 %p4, %rd1, %rd8;
+; SM90-NEXT: mov.b64 %rd8, %rd1;
; SM90-NEXT: @%p4 bra $L__BB280_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -16230,26 +16096,26 @@ define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: .reg .b64 %rd<9>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fmaximum_acq_rel_v1f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd3];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd2];
; SM90-NEXT: $L__BB284_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.eq.b64 %p1, %rd8, 0;
-; SM90-NEXT: selp.f64 %rd4, %rd8, %rd1, %p1;
-; SM90-NEXT: setp.nan.f64 %p2, %rd8, %rd1;
-; SM90-NEXT: max.f64 %rd5, %rd8, %rd1;
+; SM90-NEXT: selp.f64 %rd4, %rd8, %rd3, %p1;
+; SM90-NEXT: setp.nan.f64 %p2, %rd8, %rd3;
+; SM90-NEXT: max.f64 %rd5, %rd8, %rd3;
; SM90-NEXT: selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
-; SM90-NEXT: setp.eq.f64 %p3, %rd8, %rd1;
+; SM90-NEXT: setp.eq.f64 %p3, %rd8, %rd3;
; SM90-NEXT: selp.f64 %rd7, %rd4, %rd6, %p3;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
-; SM90-NEXT: setp.ne.b64 %p4, %rd2, %rd8;
-; SM90-NEXT: mov.b64 %rd8, %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd8, %rd7;
+; SM90-NEXT: setp.ne.b64 %p4, %rd1, %rd8;
+; SM90-NEXT: mov.b64 %rd8, %rd1;
; SM90-NEXT: @%p4 bra $L__BB284_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -16595,10 +16461,8 @@ define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v1f16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
; SM90-NEXT: st.param::func.b16 [func_retval0], %rs2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
@@ -16608,17 +16472,14 @@ define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM90-LABEL: fadd_acq_rel_v2f16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<3>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2f16_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r2, [%rd1], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -16627,19 +16488,14 @@ define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM90-LABEL: fadd_acq_rel_v4f16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<5>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4f16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4f16_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs5, [%rd1], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs2;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs3;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs4;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.v2.f16x2 {%r3, %r4}, [%rd1], {%r1, %r2};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -16648,31 +16504,13 @@ define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM90-LABEL: fadd_acq_rel_v8f16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<17>;
; SM90-NEXT: .reg .b32 %r<9>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v8f16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f16_global_cta_param_1];
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs9, [%rd1], %rs7;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+2], %rs8;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+4], %rs5;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs6;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs13, [%rd1+8], %rs3;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+10], %rs4;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs2;
-; SM90-NEXT: mov.b32 %r5, {%rs9, %rs10};
-; SM90-NEXT: mov.b32 %r6, {%rs11, %rs12};
-; SM90-NEXT: mov.b32 %r7, {%rs13, %rs14};
-; SM90-NEXT: mov.b32 %r8, {%rs15, %rs16};
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.v4.f16x2 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
@@ -16687,11 +16525,9 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v1f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs2, %rs1;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b16 [func_retval0], %rs3;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
@@ -16702,11 +16538,9 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v1f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs2, %rs1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b16 [func_retval0], %rs3;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -16717,11 +16551,9 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v1f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs2, %rs1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
; SM90-FTZ-DISALLOW-NEXT: st.param::func.b16 [func_retval0], %rs3;
; SM90-FTZ-DISALLOW-NEXT: ret;
;
@@ -16732,11 +16564,9 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v1f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs2, %rs1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
; SM90-FTZ-ALLOW-NEXT: st.param::func.b16 [func_retval0], %rs3;
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
@@ -16746,70 +16576,54 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM90-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<4>;
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fsub_acq_rel_v2f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs3, %rs1;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1], %rs3;
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs5, %rs2;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs5;
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v2.b16 [func_retval0], {%rs4, %rs6};
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r2, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<4>;
; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fsub_acq_rel_v2f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs3, %rs1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1], %rs3;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs5, %rs2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs5;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v2.b16 [func_retval0], {%rs4, %rs6};
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: neg.f16x2 %r2, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
; SM90-FTZ-DISALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<4>;
; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fsub_acq_rel_v2f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs3, %rs1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1], %rs3;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs5, %rs2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs5;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v2.b16 [func_retval0], {%rs4, %rs6};
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r2, %r1;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-FTZ-DISALLOW-NEXT: ret;
;
; SM90-FTZ-ALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<4>;
; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fsub_acq_rel_v2f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs3, %rs1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1], %rs3;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs5, %rs2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs5;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v2.b16 [func_retval0], {%rs4, %rs6};
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r2, %r1;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16x2 %r3, [%rd1], %r2;
+; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -16818,86 +16632,58 @@ define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM90-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<13>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fsub_acq_rel_v4f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs5, %rs1;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1], %rs5;
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs7, %rs2;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+2], %rs7;
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs9, %rs3;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+4], %rs9;
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs11, %rs4;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs11;
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v4.b16 [func_retval0], {%rs6, %rs8, %rs10, %rs12};
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r3, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r4, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v2.f16x2 {%r5, %r6}, [%rd1], {%r4, %r3};
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<13>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<7>;
; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fsub_acq_rel_v4f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs5, %rs1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1], %rs5;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs7, %rs2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+2], %rs7;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs9, %rs3;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+4], %rs9;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs11, %rs4;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs11;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b16 [func_retval0], {%rs6, %rs8, %rs10, %rs12};
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: neg.f16x2 %r3, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f16x2 %r4, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v2.f16x2 {%r5, %r6}, [%rd1], {%r4, %r3};
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
; SM90-FTZ-DISALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<13>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fsub_acq_rel_v4f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs5, %rs1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1], %rs5;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs7, %rs2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+2], %rs7;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs9, %rs3;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+4], %rs9;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs11, %rs4;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs11;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b16 [func_retval0], {%rs6, %rs8, %rs10, %rs12};
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r3, %r2;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r4, %r1;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v2.f16x2 {%r5, %r6}, [%rd1], {%r4, %r3};
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-FTZ-DISALLOW-NEXT: ret;
;
; SM90-FTZ-ALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<13>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<7>;
; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fsub_acq_rel_v4f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs5, %rs1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1], %rs5;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs7, %rs2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+2], %rs7;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs9, %rs3;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+4], %rs9;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs11, %rs4;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs11;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b16 [func_retval0], {%rs6, %rs8, %rs10, %rs12};
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r3, %r2;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r4, %r1;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v2.f16x2 {%r5, %r6}, [%rd1], {%r4, %r3};
+; SM90-FTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -16906,154 +16692,66 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM90-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<25>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<9>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs9, %rs7;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1], %rs9;
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs11, %rs8;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+2], %rs11;
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs13, %rs5;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+4], %rs13;
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs15, %rs6;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+6], %rs15;
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs17, %rs3;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs18, [%rd1+8], %rs17;
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs19, %rs4;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs20, [%rd1+10], %rs19;
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs21, %rs1;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs22, [%rd1+12], %rs21;
-; SM90-NOFTZ-DISALLOW-NEXT: neg.f16 %rs23, %rs2;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs24, [%rd1+14], %rs23;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r5, {%rs10, %rs12};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, {%rs14, %rs16};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, {%rs18, %rs20};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r8, {%rs22, %rs24};
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r5, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r6, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r7, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: neg.f16x2 %r8, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v4.f16x2 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<25>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<9>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<13>;
; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs9, %rs7;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1], %rs9;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs11, %rs8;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+2], %rs11;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs13, %rs5;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+4], %rs13;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs15, %rs6;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+6], %rs15;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs17, %rs3;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs18, [%rd1+8], %rs17;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs19, %rs4;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs20, [%rd1+10], %rs19;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs21, %rs1;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs22, [%rd1+12], %rs21;
-; SM90-NOFTZ-ALLOW-NEXT: neg.f16 %rs23, %rs2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs24, [%rd1+14], %rs23;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r5, {%rs10, %rs12};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r6, {%rs14, %rs16};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r7, {%rs18, %rs20};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r8, {%rs22, %rs24};
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NOFTZ-ALLOW-NEXT: neg.f16x2 %r5, %r4;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f16x2 %r6, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f16x2 %r7, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: neg.f16x2 %r8, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v4.f16x2 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
; SM90-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<25>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<9>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs9, %rs7;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1], %rs9;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs11, %rs8;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+2], %rs11;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs13, %rs5;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+4], %rs13;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs15, %rs6;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+6], %rs15;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs17, %rs3;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs18, [%rd1+8], %rs17;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs19, %rs4;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs20, [%rd1+10], %rs19;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs21, %rs1;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs22, [%rd1+12], %rs21;
-; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16 %rs23, %rs2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs24, [%rd1+14], %rs23;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r5, {%rs10, %rs12};
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r6, {%rs14, %rs16};
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r7, {%rs18, %rs20};
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r8, {%rs22, %rs24};
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r5, %r4;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r6, %r3;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r7, %r2;
+; SM90-FTZ-DISALLOW-NEXT: neg.ftz.f16x2 %r8, %r1;
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v4.f16x2 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-FTZ-DISALLOW-NEXT: ret;
;
; SM90-FTZ-ALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<25>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<9>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<13>;
; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs9, %rs7;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1], %rs9;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs11, %rs8;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+2], %rs11;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs13, %rs5;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+4], %rs13;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs15, %rs6;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+6], %rs15;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs17, %rs3;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs18, [%rd1+8], %rs17;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs19, %rs4;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs20, [%rd1+10], %rs19;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs21, %rs1;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs22, [%rd1+12], %rs21;
-; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16 %rs23, %rs2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs24, [%rd1+14], %rs23;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r5, {%rs10, %rs12};
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r6, {%rs14, %rs16};
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r7, {%rs18, %rs20};
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r8, {%rs22, %rs24};
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r5, %r4;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r6, %r3;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r7, %r2;
+; SM90-FTZ-ALLOW-NEXT: neg.ftz.f16x2 %r8, %r1;
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.v4.f16x2 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
+; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
ret <8 x half> %retval
@@ -17068,9 +16766,9 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -17105,9 +16803,9 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -17142,9 +16840,9 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -17179,9 +16877,9 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -17212,1161 +16910,249 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
}
define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v2f16_global_cta(
+; SM90-LABEL: fmin_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fmin_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.min.noftz.v2.f16 {%rs3, %rs4}, [%rd1], {%rs1, %rs2};
+; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fmin_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.min.noftz.v2.f16x2 {%r3, %r4}, [%rd1], {%r1, %r2};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fmin_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.min.noftz.v4.f16x2 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
; SM90-NOFTZ-DISALLOW: {
; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<14>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r6, 65535, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r6;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r3, %r4, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB297_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: ret;
-;
-; SM90-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v2f16_global_cta(
-; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-ALLOW-EMPTY:
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB297_1: // %atomicrmw.start
-; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r3, %r4, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB297_1;
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; SM90-NOFTZ-ALLOW-NEXT: ret;
-;
-; SM90-FTZ-DISALLOW-LABEL: fmin_acq_rel_v2f16_global_cta(
-; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-DISALLOW-EMPTY:
-; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-FTZ-DISALLOW-NEXT: $L__BB297_1: // %atomicrmw.start
-; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r3, %r4, %r2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB297_1;
-; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; SM90-FTZ-DISALLOW-NEXT: ret;
-;
-; SM90-FTZ-ALLOW-LABEL: fmin_acq_rel_v2f16_global_cta(
-; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-ALLOW-EMPTY:
-; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-FTZ-ALLOW-NEXT: $L__BB297_1: // %atomicrmw.start
-; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r3, %r4, %r2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB297_1;
-; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; SM90-FTZ-ALLOW-NEXT: ret;
- %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
- ret <2 x half> %retval
-}
-
-define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v4f16_global_cta(
-; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-NOFTZ-DISALLOW-EMPTY:
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB298_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r3, %r5, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r4, %r6, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB298_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-NOFTZ-DISALLOW-NEXT: ret;
-;
-; SM90-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v4f16_global_cta(
-; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-NOFTZ-ALLOW-EMPTY:
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB298_1: // %atomicrmw.start
-; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r3, %r5, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r4, %r6, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB298_1;
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-NOFTZ-ALLOW-NEXT: ret;
-;
-; SM90-FTZ-DISALLOW-LABEL: fmin_acq_rel_v4f16_global_cta(
-; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-FTZ-DISALLOW-EMPTY:
-; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: $L__BB298_1: // %atomicrmw.start
-; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r3, %r5, %r1;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r4, %r6, %r2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB298_1;
-; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-FTZ-DISALLOW-NEXT: ret;
-;
-; SM90-FTZ-ALLOW-LABEL: fmin_acq_rel_v4f16_global_cta(
-; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-FTZ-ALLOW-EMPTY:
-; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-FTZ-ALLOW-NEXT: $L__BB298_1: // %atomicrmw.start
-; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r3, %r5, %r1;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r4, %r6, %r2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB298_1;
-; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-FTZ-ALLOW-NEXT: ret;
- %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
- ret <4 x half> %retval
-}
-
-define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v8f16_global_cta(
-; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<26>;
-; SM90-NOFTZ-DISALLOW-EMPTY:
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 %rd2, 0;
-; SM90-NOFTZ-DISALLOW-NEXT: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: }
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB299_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r5, %r9, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd5, %r5;
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r6, %r10, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd6, %r6;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r7, %r11, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd9, %r7;
-; SM90-NOFTZ-DISALLOW-NEXT: min.f16x2 %r8, %r12, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd10, %r8;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r9;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r10;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
-; SM90-NOFTZ-DISALLOW-NEXT: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 cmp, {%rd16, %rd20};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 swap, {%rd8, %rd12};
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 {%rd21, %rd22}, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: }
-; SM90-NOFTZ-DISALLOW-NEXT: xor.b64 %rd23, %rd22, %rd20;
-; SM90-NOFTZ-DISALLOW-NEXT: xor.b64 %rd24, %rd21, %rd16;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd25, %rd24, %rd23;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd22;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd22;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd21;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd21;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB299_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-NOFTZ-DISALLOW-NEXT: ret;
-;
-; SM90-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v8f16_global_cta(
-; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<26>;
-; SM90-NOFTZ-ALLOW-EMPTY:
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 %rd2, 0;
-; SM90-NOFTZ-ALLOW-NEXT: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-NOFTZ-ALLOW-NEXT: }
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB299_1: // %atomicrmw.start
-; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r5, %r9, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd5, %r5;
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r6, %r10, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd6, %r6;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r7, %r11, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd9, %r7;
-; SM90-NOFTZ-ALLOW-NEXT: min.f16x2 %r8, %r12, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd10, %r8;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r9;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r10;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
-; SM90-NOFTZ-ALLOW-NEXT: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 cmp, {%rd16, %rd20};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 swap, {%rd8, %rd12};
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 {%rd21, %rd22}, dst;
-; SM90-NOFTZ-ALLOW-NEXT: }
-; SM90-NOFTZ-ALLOW-NEXT: xor.b64 %rd23, %rd22, %rd20;
-; SM90-NOFTZ-ALLOW-NEXT: xor.b64 %rd24, %rd21, %rd16;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd25, %rd24, %rd23;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd22;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd22;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd21;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd21;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB299_1;
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-NOFTZ-ALLOW-NEXT: ret;
-;
-; SM90-FTZ-DISALLOW-LABEL: fmin_acq_rel_v8f16_global_cta(
-; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<26>;
-; SM90-FTZ-DISALLOW-EMPTY:
-; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 %rd2, 0;
-; SM90-FTZ-DISALLOW-NEXT: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-FTZ-DISALLOW-NEXT: }
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-FTZ-DISALLOW-NEXT: $L__BB299_1: // %atomicrmw.start
-; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r5, %r9, %r1;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd5, %r5;
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r6, %r10, %r2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd6, %r6;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r7, %r11, %r3;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd9, %r7;
-; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16x2 %r8, %r12, %r4;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd10, %r8;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r9;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r10;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
-; SM90-FTZ-DISALLOW-NEXT: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 cmp, {%rd16, %rd20};
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 swap, {%rd8, %rd12};
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 {%rd21, %rd22}, dst;
-; SM90-FTZ-DISALLOW-NEXT: }
-; SM90-FTZ-DISALLOW-NEXT: xor.b64 %rd23, %rd22, %rd20;
-; SM90-FTZ-DISALLOW-NEXT: xor.b64 %rd24, %rd21, %rd16;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd25, %rd24, %rd23;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd22;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd22;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd21;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd21;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB299_1;
-; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-FTZ-DISALLOW-NEXT: ret;
-;
-; SM90-FTZ-ALLOW-LABEL: fmin_acq_rel_v8f16_global_cta(
-; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<26>;
-; SM90-FTZ-ALLOW-EMPTY:
-; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 %rd2, 0;
-; SM90-FTZ-ALLOW-NEXT: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-FTZ-ALLOW-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-FTZ-ALLOW-NEXT: }
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-FTZ-ALLOW-NEXT: $L__BB299_1: // %atomicrmw.start
-; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r5, %r9, %r1;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd5, %r5;
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r6, %r10, %r2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd6, %r6;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r7, %r11, %r3;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd9, %r7;
-; SM90-FTZ-ALLOW-NEXT: min.ftz.f16x2 %r8, %r12, %r4;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd10, %r8;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r9;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r10;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
-; SM90-FTZ-ALLOW-NEXT: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 cmp, {%rd16, %rd20};
-; SM90-FTZ-ALLOW-NEXT: mov.b128 swap, {%rd8, %rd12};
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 {%rd21, %rd22}, dst;
-; SM90-FTZ-ALLOW-NEXT: }
-; SM90-FTZ-ALLOW-NEXT: xor.b64 %rd23, %rd22, %rd20;
-; SM90-FTZ-ALLOW-NEXT: xor.b64 %rd24, %rd21, %rd16;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd25, %rd24, %rd23;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd22;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd22;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd21;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd21;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB299_1;
-; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-FTZ-ALLOW-NEXT: ret;
- %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
- ret <8 x half> %retval
-}
-
-define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
-; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<14>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
-; SM90-NOFTZ-DISALLOW-EMPTY:
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r6, 65535, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r6;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r7, %r13, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16 %rs3, %rs2, %rs1;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r10, %r13, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b32 %r11, %r10, %r9;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r13, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB300_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r12, %r3, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
-; SM90-NOFTZ-DISALLOW-NEXT: ret;
-;
-; SM90-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
-; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<14>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
-; SM90-NOFTZ-ALLOW-EMPTY:
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r6, 65535, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r6;
-; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
-; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r7, %r13, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
-; SM90-NOFTZ-ALLOW-NEXT: max.f16 %rs3, %rs2, %rs1;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r10, %r13, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: or.b32 %r11, %r10, %r9;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r13, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB300_1;
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r12, %r3, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
-; SM90-NOFTZ-ALLOW-NEXT: ret;
-;
-; SM90-FTZ-DISALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
-; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<14>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
-; SM90-FTZ-DISALLOW-EMPTY:
-; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
-; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r6, 65535, %r1;
-; SM90-FTZ-DISALLOW-NEXT: not.b32 %r2, %r6;
-; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
-; SM90-FTZ-DISALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
-; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r7, %r13, %r1;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16 %rs3, %rs2, %rs1;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
-; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, %r1;
-; SM90-FTZ-DISALLOW-NEXT: and.b32 %r10, %r13, %r2;
-; SM90-FTZ-DISALLOW-NEXT: or.b32 %r11, %r10, %r9;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r13, %r3;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB300_1;
-; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r12, %r3, %r1;
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
-; SM90-FTZ-DISALLOW-NEXT: ret;
-;
-; SM90-FTZ-ALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
-; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<14>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
-; SM90-FTZ-ALLOW-EMPTY:
-; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
-; SM90-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-FTZ-ALLOW-NEXT: shl.b32 %r6, 65535, %r1;
-; SM90-FTZ-ALLOW-NEXT: not.b32 %r2, %r6;
-; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
-; SM90-FTZ-ALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
-; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: shr.u32 %r7, %r13, %r1;
-; SM90-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16 %rs3, %rs2, %rs1;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
-; SM90-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, %r1;
-; SM90-FTZ-ALLOW-NEXT: and.b32 %r10, %r13, %r2;
-; SM90-FTZ-ALLOW-NEXT: or.b32 %r11, %r10, %r9;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r13, %r3;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB300_1;
-; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-ALLOW-NEXT: shr.u32 %r12, %r3, %r1;
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
-; SM90-FTZ-ALLOW-NEXT: ret;
- %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
- ret <1 x half> %retval
-}
-
-define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v2f16_global_cta(
-; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-DISALLOW-EMPTY:
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB301_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r3, %r4, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB301_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: ret;
-;
-; SM90-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v2f16_global_cta(
-; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-ALLOW-EMPTY:
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB301_1: // %atomicrmw.start
-; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r3, %r4, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB301_1;
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; SM90-NOFTZ-ALLOW-NEXT: ret;
-;
-; SM90-FTZ-DISALLOW-LABEL: fmax_acq_rel_v2f16_global_cta(
-; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-DISALLOW-EMPTY:
-; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-FTZ-DISALLOW-NEXT: $L__BB301_1: // %atomicrmw.start
-; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r3, %r4, %r2;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB301_1;
-; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; SM90-FTZ-DISALLOW-NEXT: ret;
-;
-; SM90-FTZ-ALLOW-LABEL: fmax_acq_rel_v2f16_global_cta(
-; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-ALLOW-EMPTY:
-; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-FTZ-ALLOW-NEXT: $L__BB301_1: // %atomicrmw.start
-; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r3, %r4, %r2;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB301_1;
-; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; SM90-FTZ-ALLOW-NEXT: ret;
- %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
- ret <2 x half> %retval
-}
-
-define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v4f16_global_cta(
-; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-NOFTZ-DISALLOW-EMPTY:
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB302_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r3, %r5, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r4, %r6, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB302_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-NOFTZ-DISALLOW-NEXT: ret;
-;
-; SM90-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v4f16_global_cta(
-; SM90-NOFTZ-ALLOW: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-NOFTZ-ALLOW-EMPTY:
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
-; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB302_1: // %atomicrmw.start
-; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r3, %r5, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r4, %r6, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB302_1;
-; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-NOFTZ-ALLOW-NEXT: ret;
-;
-; SM90-FTZ-DISALLOW-LABEL: fmax_acq_rel_v4f16_global_cta(
-; SM90-FTZ-DISALLOW: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-FTZ-DISALLOW-EMPTY:
-; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
-; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-FTZ-DISALLOW-NEXT: $L__BB302_1: // %atomicrmw.start
-; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r3, %r5, %r1;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r4, %r6, %r2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB302_1;
-; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-FTZ-DISALLOW-NEXT: ret;
-;
-; SM90-FTZ-ALLOW-LABEL: fmax_acq_rel_v4f16_global_cta(
-; SM90-FTZ-ALLOW: {
-; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<7>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<12>;
-; SM90-FTZ-ALLOW-EMPTY:
-; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
-; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-FTZ-ALLOW-NEXT: $L__BB302_1: // %atomicrmw.start
-; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r3, %r5, %r1;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r4, %r6, %r2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB302_1;
-; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-FTZ-ALLOW-NEXT: ret;
- %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
- ret <4 x half> %retval
-}
-
-define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
-; SM90-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v8f16_global_cta(
-; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<26>;
-; SM90-NOFTZ-DISALLOW-EMPTY:
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 %rd2, 0;
-; SM90-NOFTZ-DISALLOW-NEXT: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: }
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB303_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r5, %r9, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd5, %r5;
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r6, %r10, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd6, %r6;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r7, %r11, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd9, %r7;
-; SM90-NOFTZ-DISALLOW-NEXT: max.f16x2 %r8, %r12, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd10, %r8;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r9;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r10;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
-; SM90-NOFTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
-; SM90-NOFTZ-DISALLOW-NEXT: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 cmp, {%rd16, %rd20};
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 swap, {%rd8, %rd12};
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b128 {%rd21, %rd22}, dst;
-; SM90-NOFTZ-DISALLOW-NEXT: }
-; SM90-NOFTZ-DISALLOW-NEXT: xor.b64 %rd23, %rd22, %rd20;
-; SM90-NOFTZ-DISALLOW-NEXT: xor.b64 %rd24, %rd21, %rd16;
-; SM90-NOFTZ-DISALLOW-NEXT: or.b64 %rd25, %rd24, %rd23;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd22;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd22;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd21;
-; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd21;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB303_1;
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r7, %r13, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM90-NOFTZ-DISALLOW-NEXT: max.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: or.b32 %r11, %r10, %r9;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r13, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB300_1;
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r12, %r3, %r1;
; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM90-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v8f16_global_cta(
+; SM90-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
; SM90-NOFTZ-ALLOW: {
; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<26>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<14>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 %rd2, 0;
-; SM90-NOFTZ-ALLOW-NEXT: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-NOFTZ-ALLOW-NEXT: }
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NOFTZ-ALLOW-NEXT: $L__BB303_1: // %atomicrmw.start
+; SM90-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r6, 65535, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r6;
+; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-NOFTZ-ALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r5, %r9, %r1;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd5, %r5;
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r6, %r10, %r2;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd6, %r6;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r7, %r11, %r3;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd9, %r7;
-; SM90-NOFTZ-ALLOW-NEXT: max.f16x2 %r8, %r12, %r4;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd10, %r8;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r9;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r10;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
-; SM90-NOFTZ-ALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
-; SM90-NOFTZ-ALLOW-NEXT: {
-; SM90-NOFTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 cmp, {%rd16, %rd20};
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 swap, {%rd8, %rd12};
-; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b128 {%rd21, %rd22}, dst;
-; SM90-NOFTZ-ALLOW-NEXT: }
-; SM90-NOFTZ-ALLOW-NEXT: xor.b64 %rd23, %rd22, %rd20;
-; SM90-NOFTZ-ALLOW-NEXT: xor.b64 %rd24, %rd21, %rd16;
-; SM90-NOFTZ-ALLOW-NEXT: or.b64 %rd25, %rd24, %rd23;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd22;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd22;
-; SM90-NOFTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd21;
-; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd21;
-; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB303_1;
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r7, %r13, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM90-NOFTZ-ALLOW-NEXT: max.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: or.b32 %r11, %r10, %r9;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r13, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB300_1;
; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r12, %r3, %r1;
; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
-; SM90-FTZ-DISALLOW-LABEL: fmax_acq_rel_v8f16_global_cta(
+; SM90-FTZ-DISALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
; SM90-FTZ-DISALLOW: {
; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<26>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<14>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 %rd2, 0;
-; SM90-FTZ-DISALLOW-NEXT: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-FTZ-DISALLOW-NEXT: }
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-FTZ-DISALLOW-NEXT: $L__BB303_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r6, 65535, %r1;
+; SM90-FTZ-DISALLOW-NEXT: not.b32 %r2, %r6;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r5, %r9, %r1;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd5, %r5;
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r6, %r10, %r2;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd6, %r6;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r7, %r11, %r3;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd9, %r7;
-; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16x2 %r8, %r12, %r4;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd10, %r8;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd13, %r9;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd14, %r10;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
-; SM90-FTZ-DISALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
-; SM90-FTZ-DISALLOW-NEXT: {
-; SM90-FTZ-DISALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 cmp, {%rd16, %rd20};
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 swap, {%rd8, %rd12};
-; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-DISALLOW-NEXT: mov.b128 {%rd21, %rd22}, dst;
-; SM90-FTZ-DISALLOW-NEXT: }
-; SM90-FTZ-DISALLOW-NEXT: xor.b64 %rd23, %rd22, %rd20;
-; SM90-FTZ-DISALLOW-NEXT: xor.b64 %rd24, %rd21, %rd16;
-; SM90-FTZ-DISALLOW-NEXT: or.b64 %rd25, %rd24, %rd23;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r12}, %rd22;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r11, %rd22;
-; SM90-FTZ-DISALLOW-NEXT: mov.b64 {_, %r10}, %rd21;
-; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r9, %rd21;
-; SM90-FTZ-DISALLOW-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB303_1;
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r7, %r13, %r1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-FTZ-DISALLOW-NEXT: or.b32 %r11, %r10, %r9;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r13, %r3;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB300_1;
; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r12, %r3, %r1;
; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-DISALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-FTZ-DISALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
; SM90-FTZ-DISALLOW-NEXT: ret;
;
-; SM90-FTZ-ALLOW-LABEL: fmax_acq_rel_v8f16_global_cta(
+; SM90-FTZ-ALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
; SM90-FTZ-ALLOW: {
; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<13>;
-; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<26>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<14>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 %rd2, 0;
-; SM90-FTZ-ALLOW-NEXT: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-FTZ-ALLOW-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-FTZ-ALLOW-NEXT: }
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-FTZ-ALLOW-NEXT: $L__BB303_1: // %atomicrmw.start
+; SM90-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r6, 65535, %r1;
+; SM90-FTZ-ALLOW-NEXT: not.b32 %r2, %r6;
+; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-FTZ-ALLOW-NEXT: $L__BB300_1: // %atomicrmw.start
; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r5, %r9, %r1;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd5, %r5;
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r6, %r10, %r2;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd6, %r6;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r7, %r11, %r3;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd9, %r7;
-; SM90-FTZ-ALLOW-NEXT: max.ftz.f16x2 %r8, %r12, %r4;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd10, %r8;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd13, %r9;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd14, %r10;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM90-FTZ-ALLOW-NEXT: cvt.u64.u32 %rd18, %r12;
-; SM90-FTZ-ALLOW-NEXT: shl.b64 %rd19, %rd18, 32;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd20, %rd17, %rd19;
-; SM90-FTZ-ALLOW-NEXT: {
-; SM90-FTZ-ALLOW-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 cmp, {%rd16, %rd20};
-; SM90-FTZ-ALLOW-NEXT: mov.b128 swap, {%rd8, %rd12};
-; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-FTZ-ALLOW-NEXT: mov.b128 {%rd21, %rd22}, dst;
-; SM90-FTZ-ALLOW-NEXT: }
-; SM90-FTZ-ALLOW-NEXT: xor.b64 %rd23, %rd22, %rd20;
-; SM90-FTZ-ALLOW-NEXT: xor.b64 %rd24, %rd21, %rd16;
-; SM90-FTZ-ALLOW-NEXT: or.b64 %rd25, %rd24, %rd23;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r12}, %rd22;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r11, %rd22;
-; SM90-FTZ-ALLOW-NEXT: mov.b64 {_, %r10}, %rd21;
-; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r9, %rd21;
-; SM90-FTZ-ALLOW-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB303_1;
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r7, %r13, %r1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM90-FTZ-ALLOW-NEXT: max.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u16 %r8, %rs3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-FTZ-ALLOW-NEXT: or.b32 %r11, %r10, %r9;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r13;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r13, %r3;
+; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB300_1;
; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r12, %r3, %r1;
; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
-; SM90-FTZ-ALLOW-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-FTZ-ALLOW-NEXT: st.param::func.b16 [func_retval0], %r12;
; SM90-FTZ-ALLOW-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fmax_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.max.noftz.v2.f16 {%rs3, %rs4}, [%rd1], {%rs1, %rs2};
+; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.max.noftz.v2.f16x2 {%r3, %r4}, [%rd1], {%r1, %r2};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.max.noftz.v4.f16x2 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
ret <8 x half> %retval
}
@@ -18380,9 +17166,9 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -18417,9 +17203,9 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -18454,9 +17240,9 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -18491,9 +17277,9 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -19036,9 +17822,9 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -19073,9 +17859,9 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-NOFTZ-ALLOW-EMPTY:
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM90-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -19110,9 +17896,9 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-FTZ-DISALLOW-EMPTY:
; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-DISALLOW-NEXT: ld.param::func.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -19147,9 +17933,9 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM90-FTZ-ALLOW-EMPTY:
; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: ld.param::func.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
-; SM90-FTZ-ALLOW-NEXT: ld.param::func.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM90-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
@@ -19691,10 +18477,8 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v1bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs2, [%rd1], %rs1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.bf16 %rs2, [%rd1], %rs1;
; SM90-NEXT: st.param::func.b16 [func_retval0], %rs2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
@@ -19704,17 +18488,14 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fadd_acq_rel_v2bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<3>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2bf16_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs3, [%rd1], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs4, [%rd1+2], %rs2;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.bf16x2 %r2, [%rd1], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -19723,19 +18504,14 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM90-LABEL: fadd_acq_rel_v4bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<5>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4bf16_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs5, [%rd1], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs6, [%rd1+2], %rs2;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs7, [%rd1+4], %rs3;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs8, [%rd1+6], %rs4;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.v2.bf16x2 {%r3, %r4}, [%rd1], {%r1, %r2};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -19744,31 +18520,13 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM90-LABEL: fadd_acq_rel_v8bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<17>;
; SM90-NEXT: .reg .b32 %r<9>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs9, [%rd1], %rs7;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs10, [%rd1+2], %rs8;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs11, [%rd1+4], %rs5;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs12, [%rd1+6], %rs6;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs13, [%rd1+8], %rs3;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs14, [%rd1+10], %rs4;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs15, [%rd1+12], %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs16, [%rd1+14], %rs2;
-; SM90-NEXT: mov.b32 %r5, {%rs9, %rs10};
-; SM90-NEXT: mov.b32 %r6, {%rs11, %rs12};
-; SM90-NEXT: mov.b32 %r7, {%rs13, %rs14};
-; SM90-NEXT: mov.b32 %r8, {%rs15, %rs16};
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.v4.bf16x2 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
@@ -19783,11 +18541,9 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v1bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: neg.bf16 %rs2, %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs3, [%rd1], %rs2;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.bf16 %rs3, [%rd1], %rs2;
; SM90-NEXT: st.param::func.b16 [func_retval0], %rs3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
@@ -19797,19 +18553,15 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fsub_acq_rel_v2bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<7>;
+; SM90-NEXT: .reg .b32 %r<4>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fsub_acq_rel_v2bf16_global_cta_param_1];
-; SM90-NEXT: neg.bf16 %rs3, %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs4, [%rd1], %rs3;
-; SM90-NEXT: neg.bf16 %rs5, %rs2;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs6, [%rd1+2], %rs5;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs4, %rs6};
+; SM90-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: neg.bf16x2 %r2, %r1;
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.bf16x2 %r3, [%rd1], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -19818,23 +18570,16 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM90-LABEL: fsub_acq_rel_v4bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<7>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v4bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fsub_acq_rel_v4bf16_global_cta_param_1];
-; SM90-NEXT: neg.bf16 %rs5, %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs6, [%rd1], %rs5;
-; SM90-NEXT: neg.bf16 %rs7, %rs2;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs8, [%rd1+2], %rs7;
-; SM90-NEXT: neg.bf16 %rs9, %rs3;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs10, [%rd1+4], %rs9;
-; SM90-NEXT: neg.bf16 %rs11, %rs4;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs12, [%rd1+6], %rs11;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v4.b16 [func_retval0], {%rs6, %rs8, %rs10, %rs12};
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: neg.bf16x2 %r3, %r2;
+; SM90-NEXT: neg.bf16x2 %r4, %r1;
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.v2.bf16x2 {%r5, %r6}, [%rd1], {%r4, %r3};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -19843,40 +18588,18 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM90-LABEL: fsub_acq_rel_v8bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<25>;
-; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b32 %r<13>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM90-NEXT: neg.bf16 %rs9, %rs7;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs10, [%rd1], %rs9;
-; SM90-NEXT: neg.bf16 %rs11, %rs8;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs12, [%rd1+2], %rs11;
-; SM90-NEXT: neg.bf16 %rs13, %rs5;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs14, [%rd1+4], %rs13;
-; SM90-NEXT: neg.bf16 %rs15, %rs6;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs16, [%rd1+6], %rs15;
-; SM90-NEXT: neg.bf16 %rs17, %rs3;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs18, [%rd1+8], %rs17;
-; SM90-NEXT: neg.bf16 %rs19, %rs4;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs20, [%rd1+10], %rs19;
-; SM90-NEXT: neg.bf16 %rs21, %rs1;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs22, [%rd1+12], %rs21;
-; SM90-NEXT: neg.bf16 %rs23, %rs2;
-; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs24, [%rd1+14], %rs23;
-; SM90-NEXT: mov.b32 %r5, {%rs10, %rs12};
-; SM90-NEXT: mov.b32 %r6, {%rs14, %rs16};
-; SM90-NEXT: mov.b32 %r7, {%rs18, %rs20};
-; SM90-NEXT: mov.b32 %r8, {%rs22, %rs24};
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: neg.bf16x2 %r5, %r4;
+; SM90-NEXT: neg.bf16x2 %r6, %r3;
+; SM90-NEXT: neg.bf16x2 %r7, %r2;
+; SM90-NEXT: neg.bf16x2 %r8, %r1;
+; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.v4.bf16x2 {%r9, %r10, %r11, %r12}, [%rd1], {%r8, %r7, %r6, %r5};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -19891,9 +18614,9 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -19926,25 +18649,14 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fmin_acq_rel_v2bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b16 %rs<5>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB321_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: min.bf16x2 %r3, %r4, %r2;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB321_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fmin_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.min.noftz.v2.bf16 {%rs3, %rs4}, [%rd1], {%rs1, %rs2};
+; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -19953,37 +18665,14 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM90-LABEL: fmin_acq_rel_v4bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<7>;
-; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB322_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: min.bf16x2 %r3, %r5, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-NEXT: min.bf16x2 %r4, %r6, %r2;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB322_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.min.noftz.v2.bf16x2 {%r3, %r4}, [%rd1], {%r1, %r2};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -19992,67 +18681,14 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM90-LABEL: fmin_acq_rel_v8bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<13>;
-; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: mov.b64 %rd2, 0;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB323_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: min.bf16x2 %r5, %r9, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
-; SM90-NEXT: min.bf16x2 %r6, %r10, %r2;
-; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
-; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-NEXT: min.bf16x2 %r7, %r11, %r3;
-; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
-; SM90-NEXT: min.bf16x2 %r8, %r12, %r4;
-; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
-; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
-; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
-; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
-; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
-; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
-; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
-; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
-; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
-; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
-; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB323_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.min.noftz.v4.bf16x2 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -20067,9 +18703,9 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -20102,25 +18738,14 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fmax_acq_rel_v2bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b16 %rs<5>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB325_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: max.bf16x2 %r3, %r4, %r2;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB325_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fmax_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.max.noftz.v2.bf16 {%rs3, %rs4}, [%rd1], {%rs1, %rs2};
+; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -20129,37 +18754,14 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM90-LABEL: fmax_acq_rel_v4bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<7>;
-; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB326_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: max.bf16x2 %r3, %r5, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM90-NEXT: max.bf16x2 %r4, %r6, %r2;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB326_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.max.noftz.v2.bf16x2 {%r3, %r4}, [%rd1], {%r1, %r2};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -20168,67 +18770,14 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM90-LABEL: fmax_acq_rel_v8bf16_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<13>;
-; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: mov.b64 %rd2, 0;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB327_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: max.bf16x2 %r5, %r9, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
-; SM90-NEXT: max.bf16x2 %r6, %r10, %r2;
-; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
-; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-NEXT: max.bf16x2 %r7, %r11, %r3;
-; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
-; SM90-NEXT: max.bf16x2 %r8, %r12, %r4;
-; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
-; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
-; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
-; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
-; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
-; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
-; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
-; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
-; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
-; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
-; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
-; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB327_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.max.noftz.v4.bf16x2 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -20243,9 +18792,9 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -20419,9 +18968,9 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -20595,14 +19144,14 @@ define <1 x i8> @add_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd2, [add_monotonic_v1i8_global_cta_param_0];
-; SM90-NEXT: ld.param::func.b8 %r6, [add_monotonic_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: and.b32 %r8, %r7, 3;
-; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
; SM90-NEXT: shl.b32 %r2, 255, %r1;
; SM90-NEXT: not.b32 %r3, %r2;
-; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.param::func.b8 %r8, [add_monotonic_v1i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r4, %r8, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM90-NEXT: $L__BB336_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -20819,14 +19368,14 @@ define <1 x i8> @add_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd2, [add_acquire_v1i8_global_cta_param_0];
-; SM90-NEXT: ld.param::func.b8 %r6, [add_acquire_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: and.b32 %r8, %r7, 3;
-; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
; SM90-NEXT: shl.b32 %r2, 255, %r1;
; SM90-NEXT: not.b32 %r3, %r2;
-; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.param::func.b8 %r8, [add_acquire_v1i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r4, %r8, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM90-NEXT: $L__BB340_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -21048,14 +19597,14 @@ define <1 x i8> @add_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd2, [add_release_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b8 %r6, [add_release_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: and.b32 %r8, %r7, 3;
-; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
; SM90-NEXT: shl.b32 %r2, 255, %r1;
; SM90-NEXT: not.b32 %r3, %r2;
-; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.param::func.b8 %r8, [add_release_v1i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r4, %r8, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM90-NEXT: $L__BB344_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -21352,8 +19901,7 @@ define <1 x i32> @add_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [add_acquire_v1i32_global_cta_param_0];
; SM90-NEXT: ld.param::func.b32 %r1, [add_acquire_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
-; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: atom.acquire.cta.global.add.u32 %r2, [%rd1], %r1;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
@@ -21432,9 +19980,8 @@ define <1 x i32> @add_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd1, [add_release_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param::func.b32 %r1, [add_release_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: atom.release.cta.global.add.u32 %r2, [%rd1], %r1;
; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
@@ -21514,14 +20061,14 @@ define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd2, [nand_monotonic_v1i8_global_cta_param_0];
-; SM90-NEXT: ld.param::func.b8 %r6, [nand_monotonic_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: and.b32 %r8, %r7, 3;
-; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
; SM90-NEXT: shl.b32 %r2, 255, %r1;
; SM90-NEXT: not.b32 %r3, %r2;
-; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.param::func.b8 %r8, [nand_monotonic_v1i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r4, %r8, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM90-NEXT: $L__BB360_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -21653,14 +20200,14 @@ define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd2, [nand_acquire_v1i8_global_cta_param_0];
-; SM90-NEXT: ld.param::func.b8 %r6, [nand_acquire_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: and.b32 %r8, %r7, 3;
-; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
; SM90-NEXT: shl.b32 %r2, 255, %r1;
; SM90-NEXT: not.b32 %r3, %r2;
-; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.param::func.b8 %r8, [nand_acquire_v1i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r4, %r8, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM90-NEXT: $L__BB364_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -21797,14 +20344,14 @@ define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param::func.b64 %rd2, [nand_release_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b8 %r6, [nand_release_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: and.b32 %r8, %r7, 3;
-; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
; SM90-NEXT: shl.b32 %r2, 255, %r1;
; SM90-NEXT: not.b32 %r3, %r2;
-; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.param::func.b8 %r8, [nand_release_v1i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r4, %r8, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM90-NEXT: $L__BB368_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -21938,19 +20485,19 @@ define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b32 %r2, [nand_monotonic_v1i32_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
-; SM90-NEXT: ld.param::func.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB372_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
; SM90-NEXT: not.b32 %r4, %r3;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
; SM90-NEXT: @%p1 bra $L__BB372_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
ret <1 x i32> %retval
@@ -22195,20 +20742,20 @@ define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b32 %r2, [nand_acquire_v1i32_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
-; SM90-NEXT: ld.param::func.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB376_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
; SM90-NEXT: not.b32 %r4, %r3;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
; SM90-NEXT: @%p1 bra $L__BB376_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
ret <1 x i32> %retval
@@ -22456,20 +21003,20 @@ define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param::func.b32 %r2, [nand_release_v1i32_global_cta_param_1];
; SM90-NEXT: ld.param::func.b64 %rd1, [nand_release_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param::func.b32 %r1, [nand_release_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB380_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
; SM90-NEXT: not.b32 %r4, %r3;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
; SM90-NEXT: @%p1 bra $L__BB380_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
ret <1 x i32> %retval
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll b/llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll
index 44f2645c145691..40ee54c2805153 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll
@@ -68,6 +68,101 @@ define i128 @atomicrmw_xchg_l2_cache_policy_b128(ptr addrspace(1) %p, i128 %v) {
ret i128 %old
}
+define <4 x float> @atomicrmw_fadd_v4f32_l2_cache_policy(ptr addrspace(1) %p, <4 x float> %v) {
+; CHECK-LABEL: atomicrmw_fadd_v4f32_l2_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: atom.relaxed.sys.global.add.L2::cache_hint.v4.f32 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4}, %rd2;
+ %old = atomicrmw elementwise fadd ptr addrspace(1) %p, <4 x float> %v monotonic, align 16, !mem.cache_hint !1
+ ret <4 x float> %old
+}
+
+; IR volatility does not restrict cache policies on PTX atom instructions.
+define <4 x float> @atomicrmw_fadd_v4f32_volatile_l2_cache_policy(ptr addrspace(1) %p, <4 x float> %v) {
+; CHECK-LABEL: atomicrmw_fadd_v4f32_volatile_l2_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: atom.relaxed.sys.global.add.L2::cache_hint.v4.f32 {%r5, %r6, %r7, %r8}, [%rd1], {%r1, %r2, %r3, %r4}, %rd2;
+ %old = atomicrmw volatile elementwise fadd ptr addrspace(1) %p, <4 x float> %v monotonic, align 16, !mem.cache_hint !1
+ ret <4 x float> %old
+}
+
+define <4 x half> @atomicrmw_fmin_v4f16_l2_cache_policy(ptr addrspace(1) %p, <4 x half> %v) {
+; CHECK-LABEL: atomicrmw_fmin_v4f16_l2_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: atom.relaxed.sys.global.min.noftz.L2::cache_hint.v2.f16x2 {%r3, %r4}, [%rd1], {%r1, %r2}, %rd2;
+ %old = atomicrmw elementwise fmin ptr addrspace(1) %p, <4 x half> %v monotonic, align 8, !mem.cache_hint !1
+ ret <4 x half> %old
+}
+
+define <2 x bfloat> @atomicrmw_fmax_v2bf16_l2_cache_policy(ptr addrspace(1) %p, <2 x bfloat> %v) {
+; CHECK-LABEL: atomicrmw_fmax_v2bf16_l2_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: atom.relaxed.sys.global.max.noftz.L2::cache_hint.v2.bf16 {%rs3, %rs4}, [%rd1], {%rs1, %rs2}, %rd2;
+ %old = atomicrmw elementwise fmax ptr addrspace(1) %p, <2 x bfloat> %v monotonic, align 4, !mem.cache_hint !1
+ ret <2 x bfloat> %old
+}
+
+; Splitting an elementwise atomic preserves the cache policy on each piece.
+define <8 x float> @atomicrmw_fadd_v8f32_l2_cache_policy(ptr addrspace(1) %p, <8 x float> %v) {
+; CHECK-LABEL: atomicrmw_fadd_v8f32_l2_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: atom.relaxed.sys.global.add.L2::cache_hint.v4.f32 {%r9, %r10, %r11, %r12}, [%rd1], {%r5, %r6, %r7, %r8}, %rd2;
+; CHECK: atom.relaxed.sys.global.add.L2::cache_hint.v4.f32 {%r13, %r14, %r15, %r16}, [%rd1+16], {%r1, %r2, %r3, %r4}, %rd2;
+ %old = atomicrmw elementwise fadd ptr addrspace(1) %p, <8 x float> %v monotonic, align 32, !mem.cache_hint !1
+ ret <8 x float> %old
+}
+
+; Cmpxchg expansion drops cache policies unsupported by atom.cas.
+define <2 x i32> @atomicrmw_nand_v2i32_l2_cache_policy(ptr addrspace(1) %p, <2 x i32> %v) {
+; CHECK-LABEL: atomicrmw_nand_v2i32_l2_cache_policy(
+; CHECK: ld.relaxed.sys.global.b64 %rd5, [%rd2];
+; CHECK: mov.b64 {%r1, %r2}, %rd3;
+; CHECK: mov.b64 {%r3, %r4}, %rd5;
+; CHECK: mov.b64 %rd4, {%r8, %r7};
+; CHECK: atom.relaxed.sys.global.cas.b64 %rd1, [%rd2], %rd5, %rd4;
+; CHECK: mov.b64 %rd5, %rd1;
+ %old = atomicrmw elementwise nand ptr addrspace(1) %p, <2 x i32> %v monotonic, align 8, !mem.cache_hint !1
+ ret <2 x i32> %old
+}
+
+define i32 @atomicrmw_add_volatile_l2_cache_policy(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: atomicrmw_add_volatile_l2_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: atom.relaxed.sys.global.add.L2::cache_hint.u32 %r2, [%rd1], %r1, %rd2;
+ %old = atomicrmw volatile add ptr addrspace(1) %p, i32 %v monotonic, align 4, !mem.cache_hint !1
+ ret i32 %old
+}
+
+; Relaxed volatile loads/stores use MMIO forms, which cannot take cache policies.
+define i32 @atomic_load_store_volatile_l2_cache_policy(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: atomic_load_store_volatile_l2_cache_policy(
+; CHECK: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK: st.mmio.relaxed.sys.global.b32 [%rd1], %r2;
+ %old = load atomic volatile i32, ptr addrspace(1) %p monotonic, align 4, !mem.cache_hint !3
+ store atomic volatile i32 %v, ptr addrspace(1) %p monotonic, align 4, !mem.cache_hint !2
+ ret i32 %old
+}
+
+; Acquire/release forms drop the PTX volatile modifier and can retain hints.
+define i32 @atomic_load_store_volatile_ordered_cache_policy(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: atomic_load_store_volatile_ordered_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.acquire.sys.global.L1::evict_first.L2::cache_hint.L2::128B.b32 %r1, [%rd1], %rd2;
+; CHECK: st.release.sys.global.L2::cache_hint.b32 [%rd1], %r2, %rd2;
+ %old = load atomic volatile i32, ptr addrspace(1) %p acquire, align 4, !mem.cache_hint !3
+ store atomic volatile i32 %v, ptr addrspace(1) %p release, align 4, !mem.cache_hint !2
+ ret i32 %old
+}
+
+; PTX volatile loads support prefetch sizes but not eviction or cache policies.
+define i32 @load_store_volatile_cache_policy(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: load_store_volatile_cache_policy(
+; CHECK: ld.volatile.global.L2::128B.b32 %r1, [%rd1];
+; CHECK: st.volatile.global.b32 [%rd1], %r2;
+ %old = load volatile i32, ptr addrspace(1) %p, align 4, !mem.cache_hint !3
+ store volatile i32 %v, ptr addrspace(1) %p, align 4, !mem.cache_hint !2
+ ret i32 %old
+}
+
!0 = !{i32 0, !10}
!1 = !{i32 0, !11}
!2 = !{i32 1, !11}
diff --git a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
index 2a1b2856b66bdb..fb791439fdbf3b 100644
--- a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
+++ b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
@@ -1,16 +1,15 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=nvptx64-nvidia-cuda -mcpu=sm_90 -mattr=+ptx83 | FileCheck %s
+; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=nvptx64-nvidia-cuda -mcpu=sm_90 -mattr=+ptx83 -nvptx-allow-ftz-atomics=false | FileCheck %s
target triple = "nvptx64-nvidia-cuda"
-; A <1 x T> elementwise atomicrmw collapses directly to a scalar RMW.
+; Native <1 x T> elementwise atomicrmw is preserved for backend legalization.
; Shared-memory f32 atom.add preserves denormals, matching the default mode.
define <1 x float> @fadd_v1f32_elementwise(ptr addrspace(3) %addr, <1 x float> %val) {
; CHECK-LABEL: @fadd_v1f32_elementwise(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LANE_VAL:%.*]] = extractelement <1 x float> [[VAL:%.*]], i64 0
-; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw fadd ptr addrspace(3) [[ADDR:%.*]], float [[LANE_VAL]] monotonic, align 4
-; CHECK-NEXT: [[LANE_OLD:%.*]] = insertelement <1 x float> poison, float [[TMP0]], i64 0
+; CHECK-NEXT: [[LANE_OLD:%.*]] = atomicrmw elementwise fadd ptr addrspace(3) [[ADDR:%.*]], <1 x float> [[VAL:%.*]] monotonic, align 4
; CHECK-NEXT: ret <1 x float> [[LANE_OLD]]
;
entry:
@@ -18,19 +17,15 @@ entry:
ret <1 x float> %old
}
-; A <2 x float> elementwise fadd in shared memory splits into native scalar atomics.
+; Shared-memory fadd splits into one-lane atomics for backend legalization.
define <2 x float> @fadd_v2f32_elementwise(ptr addrspace(3) %addr, <2 x float> %val) {
; CHECK-LABEL: @fadd_v2f32_elementwise(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[LO_VAL1:%.*]] = shufflevector <2 x float> [[VAL:%.*]], <2 x float> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[HI_VAL1:%.*]] = shufflevector <2 x float> [[VAL]], <2 x float> poison, <1 x i32> <i32 1>
; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds <1 x float>, ptr addrspace(3) [[ADDR:%.*]], i64 1
-; CHECK-NEXT: [[LO_VAL:%.*]] = extractelement <1 x float> [[LO_VAL1]], i64 0
-; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw fadd ptr addrspace(3) [[ADDR]], float [[LO_VAL]] monotonic, align 8
-; CHECK-NEXT: [[LANE_OLD:%.*]] = insertelement <1 x float> poison, float [[TMP0]], i64 0
-; CHECK-NEXT: [[HI_VAL:%.*]] = extractelement <1 x float> [[HI_VAL1]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr addrspace(3) [[HI_PTR]], float [[HI_VAL]] monotonic, align 4
-; CHECK-NEXT: [[LANE_OLD3:%.*]] = insertelement <1 x float> poison, float [[TMP1]], i64 0
+; CHECK-NEXT: [[LANE_OLD:%.*]] = atomicrmw elementwise fadd ptr addrspace(3) [[ADDR]], <1 x float> [[LO_VAL1]] monotonic, align 8
+; CHECK-NEXT: [[LANE_OLD3:%.*]] = atomicrmw elementwise fadd ptr addrspace(3) [[HI_PTR]], <1 x float> [[HI_VAL1]] monotonic, align 4
; CHECK-NEXT: [[HI_OLD:%.*]] = shufflevector <1 x float> [[LANE_OLD]], <1 x float> [[LANE_OLD3]], <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: ret <2 x float> [[HI_OLD]]
;
@@ -46,12 +41,8 @@ define <2 x i32> @add_metadata_v2i32_elementwise(ptr %addr, <2 x i32> %val) {
; CHECK-NEXT: [[LO_VAL:%.*]] = shufflevector <2 x i32> [[VAL:%.*]], <2 x i32> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[HI_VAL:%.*]] = shufflevector <2 x i32> [[VAL]], <2 x i32> poison, <1 x i32> <i32 1>
; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds <1 x i32>, ptr [[ADDR:%.*]], i64 1
-; CHECK-NEXT: [[LANE_VAL:%.*]] = extractelement <1 x i32> [[LO_VAL]], i64 0
-; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[LANE_VAL]] monotonic, align 8, !mmra [[META0:![0-9]+]]
-; CHECK-NEXT: [[LANE_OLD:%.*]] = insertelement <1 x i32> poison, i32 [[TMP0]], i64 0
-; CHECK-NEXT: [[LANE_VAL2:%.*]] = extractelement <1 x i32> [[HI_VAL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[HI_PTR]], i32 [[LANE_VAL2]] monotonic, align 4, !mmra [[META0]]
-; CHECK-NEXT: [[LANE_OLD3:%.*]] = insertelement <1 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[LANE_OLD:%.*]] = atomicrmw elementwise add ptr [[ADDR]], <1 x i32> [[LO_VAL]] monotonic, align 8, !mmra [[META0:![0-9]+]]
+; CHECK-NEXT: [[LANE_OLD3:%.*]] = atomicrmw elementwise add ptr [[HI_PTR]], <1 x i32> [[HI_VAL]] monotonic, align 4, !mmra [[META0]]
; CHECK-NEXT: [[HI_OLD:%.*]] = shufflevector <1 x i32> [[LANE_OLD]], <1 x i32> [[LANE_OLD3]], <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: ret <2 x i32> [[HI_OLD]]
;
@@ -67,12 +58,8 @@ define <2 x float> @fadd_volatile_v2f32_elementwise(ptr addrspace(3) %addr, <2 x
; CHECK-NEXT: [[LO_VAL1:%.*]] = shufflevector <2 x float> [[VAL:%.*]], <2 x float> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[HI_VAL1:%.*]] = shufflevector <2 x float> [[VAL]], <2 x float> poison, <1 x i32> <i32 1>
; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds <1 x float>, ptr addrspace(3) [[ADDR:%.*]], i64 1
-; CHECK-NEXT: [[LO_VAL:%.*]] = extractelement <1 x float> [[LO_VAL1]], i64 0
-; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw volatile fadd ptr addrspace(3) [[ADDR]], float [[LO_VAL]] monotonic, align 8
-; CHECK-NEXT: [[LANE_OLD:%.*]] = insertelement <1 x float> poison, float [[TMP0]], i64 0
-; CHECK-NEXT: [[HI_VAL:%.*]] = extractelement <1 x float> [[HI_VAL1]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw volatile fadd ptr addrspace(3) [[HI_PTR]], float [[HI_VAL]] monotonic, align 4
-; CHECK-NEXT: [[LANE_OLD3:%.*]] = insertelement <1 x float> poison, float [[TMP1]], i64 0
+; CHECK-NEXT: [[LANE_OLD:%.*]] = atomicrmw volatile elementwise fadd ptr addrspace(3) [[ADDR]], <1 x float> [[LO_VAL1]] monotonic, align 8
+; CHECK-NEXT: [[LANE_OLD3:%.*]] = atomicrmw volatile elementwise fadd ptr addrspace(3) [[HI_PTR]], <1 x float> [[HI_VAL1]] monotonic, align 4
; CHECK-NEXT: [[HI_OLD:%.*]] = shufflevector <1 x float> [[LANE_OLD]], <1 x float> [[LANE_OLD3]], <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: ret <2 x float> [[HI_OLD]]
;
@@ -93,22 +80,14 @@ define <4 x i32> @add_acq_rel_v4i32_elementwise(ptr %addr, <4 x i32> %val) {
; CHECK-NEXT: [[LO_VAL3:%.*]] = shufflevector <2 x i32> [[LO_VAL]], <2 x i32> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[HI_VAL4:%.*]] = shufflevector <2 x i32> [[LO_VAL]], <2 x i32> poison, <1 x i32> <i32 1>
; CHECK-NEXT: [[HI_PTR4:%.*]] = getelementptr inbounds <1 x i32>, ptr [[ADDR]], i64 1
-; CHECK-NEXT: [[LO_VAL2:%.*]] = extractelement <1 x i32> [[LO_VAL3]], i64 0
-; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[LO_VAL2]] syncscope("block") monotonic, align 16
-; CHECK-NEXT: [[LANE_OLD:%.*]] = insertelement <1 x i32> poison, i32 [[TMP0]], i64 0
-; CHECK-NEXT: [[HI_VAL3:%.*]] = extractelement <1 x i32> [[HI_VAL4]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[HI_PTR4]], i32 [[HI_VAL3]] syncscope("block") monotonic, align 4
-; CHECK-NEXT: [[LANE_OLD6:%.*]] = insertelement <1 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[LANE_OLD:%.*]] = atomicrmw elementwise add ptr [[ADDR]], <1 x i32> [[LO_VAL3]] syncscope("block") monotonic, align 16
+; CHECK-NEXT: [[LANE_OLD6:%.*]] = atomicrmw elementwise add ptr [[HI_PTR4]], <1 x i32> [[HI_VAL4]] syncscope("block") monotonic, align 4
; CHECK-NEXT: [[HI_OLD:%.*]] = shufflevector <1 x i32> [[LANE_OLD]], <1 x i32> [[LANE_OLD6]], <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[LO_VAL7:%.*]] = shufflevector <2 x i32> [[HI_VAL]], <2 x i32> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[HI_VAL8:%.*]] = shufflevector <2 x i32> [[HI_VAL]], <2 x i32> poison, <1 x i32> <i32 1>
; CHECK-NEXT: [[HI_PTR7:%.*]] = getelementptr inbounds <1 x i32>, ptr [[HI_PTR]], i64 1
-; CHECK-NEXT: [[LO_VAL5:%.*]] = extractelement <1 x i32> [[LO_VAL7]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = atomicrmw add ptr [[HI_PTR]], i32 [[LO_VAL5]] syncscope("block") monotonic, align 8
-; CHECK-NEXT: [[LANE_OLD12:%.*]] = insertelement <1 x i32> poison, i32 [[TMP2]], i64 0
-; CHECK-NEXT: [[HI_VAL6:%.*]] = extractelement <1 x i32> [[HI_VAL8]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = atomicrmw add ptr [[HI_PTR7]], i32 [[HI_VAL6]] syncscope("block") monotonic, align 4
-; CHECK-NEXT: [[LANE_OLD14:%.*]] = insertelement <1 x i32> poison, i32 [[TMP3]], i64 0
+; CHECK-NEXT: [[LANE_OLD12:%.*]] = atomicrmw elementwise add ptr [[HI_PTR]], <1 x i32> [[LO_VAL7]] syncscope("block") monotonic, align 8
+; CHECK-NEXT: [[LANE_OLD14:%.*]] = atomicrmw elementwise add ptr [[HI_PTR7]], <1 x i32> [[HI_VAL8]] syncscope("block") monotonic, align 4
; CHECK-NEXT: [[HI_OLD9:%.*]] = shufflevector <1 x i32> [[LANE_OLD12]], <1 x i32> [[LANE_OLD14]], <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[OLD1:%.*]] = shufflevector <2 x i32> [[HI_OLD]], <2 x i32> [[HI_OLD9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: fence syncscope("block") acquire
@@ -306,6 +285,36 @@ entry:
ret <8 x i32> %retval
}
+; Generic pointers require recursive splitting into one-lane atomics. Preserve
+; the denormal-mode permission on each child so native f32 atomics remain usable
+; when FTZ atomics are disallowed.
+define <4 x float> @fadd_ignore_denormal_mode_v4f32_elementwise(ptr %addr, <4 x float> %val) {
+; CHECK-LABEL: @fadd_ignore_denormal_mode_v4f32_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LO_VAL:%.*]] = shufflevector <4 x float> [[VAL:%.*]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[HI_VAL:%.*]] = shufflevector <4 x float> [[VAL]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds <2 x float>, ptr [[ADDR:%.*]], i64 1
+; CHECK-NEXT: [[LO_VAL2:%.*]] = shufflevector <2 x float> [[LO_VAL]], <2 x float> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[HI_VAL3:%.*]] = shufflevector <2 x float> [[LO_VAL]], <2 x float> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[HI_PTR4:%.*]] = getelementptr inbounds <1 x float>, ptr [[ADDR]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw elementwise fadd ptr [[ADDR]], <1 x float> [[LO_VAL2]] monotonic, align 16, !atomic.ignore.denormal.mode [[META3:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise fadd ptr [[HI_PTR4]], <1 x float> [[HI_VAL3]] monotonic, align 4, !atomic.ignore.denormal.mode [[META3]]
+; CHECK-NEXT: [[OLD:%.*]] = shufflevector <1 x float> [[TMP0]], <1 x float> [[TMP1]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[LO_VAL5:%.*]] = shufflevector <2 x float> [[HI_VAL]], <2 x float> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[HI_VAL6:%.*]] = shufflevector <2 x float> [[HI_VAL]], <2 x float> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[HI_PTR7:%.*]] = getelementptr inbounds <1 x float>, ptr [[HI_PTR]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = atomicrmw elementwise fadd ptr [[HI_PTR]], <1 x float> [[LO_VAL5]] monotonic, align 8, !atomic.ignore.denormal.mode [[META3]]
+; CHECK-NEXT: [[TMP3:%.*]] = atomicrmw elementwise fadd ptr [[HI_PTR7]], <1 x float> [[HI_VAL6]] monotonic, align 4, !atomic.ignore.denormal.mode [[META3]]
+; CHECK-NEXT: [[OLD8:%.*]] = shufflevector <1 x float> [[TMP2]], <1 x float> [[TMP3]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[OLD1:%.*]] = shufflevector <2 x float> [[OLD]], <2 x float> [[OLD8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <4 x float> [[OLD1]]
+;
+entry:
+ %old = atomicrmw elementwise fadd ptr %addr, <4 x float> %val monotonic, !atomic.ignore.denormal.mode !3
+ ret <4 x float> %old
+}
+
!0 = !{!1, !2}
!1 = !{!"foo", !"bar"}
!2 = !{!"bux", !"baz"}
+!3 = !{}
More information about the llvm-branch-commits
mailing list