[llvm] [AtomicExpand][NVPTX] Add elementwise expansion and implement for NVPTX (PR #196464)
Yonah Goldberg via llvm-commits
llvm-commits at lists.llvm.org
Fri May 8 18:15:02 PDT 2026
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/196464
>From 9bd4649b8ed3905e60f4b23ce9333b2f2c36f65d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 8 May 2026 18:28:49 +0000
Subject: [PATCH 1/5] first commit
---
llvm/include/llvm/CodeGen/TargetLowering.h | 30 +-
llvm/lib/CodeGen/AtomicExpandPass.cpp | 163 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 95 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 11 +
.../NVPTX/atomicrmw-elementwise-sm60.ll | 16128 ++++++++++++++++
.../NVPTX/atomicrmw-elementwise-sm70.ll | 16076 +++++++++++++++
.../NVPTX/atomicrmw-elementwise-sm90.ll | 15129 +++++++++++++++
.../NVPTX/expand-atomic-rmw-elementwise.ll | 162 +
.../X86/expand-atomic-rmw-elementwise.ll | 72 +
9 files changed, 47832 insertions(+), 34 deletions(-)
create mode 100644 llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
create mode 100644 llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
create mode 100644 llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
create mode 100644 llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
create mode 100644 llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index f1119e4acce58..99c43a6594bf0 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -270,6 +270,19 @@ class LLVM_ABI TargetLoweringBase {
Expand, // Generic expansion in terms of other atomic operations.
CustomExpand, // Custom target-specific expansion using TLI hooks.
+ // Halve an elementwise vector atomicrmw and re-ask the target how to expand
+ // each half. Split an `atomicrmw elementwise <N x T>` into either
+ // two `atomicrmw elementwise <N/2 x T>` (when N > 2) or two scalar
+ // `atomicrmw T` (when N == 2). An `<1 x T>` elementwise RMW is collapsed
+ // directly to a scalar `atomicrmw T`. Each resulting atomicrmw is fed back through the
+ // expansion pipeline, so the target's `shouldExpandAtomicRMWInIR` is
+ // re-queried at the halved width and may return any expansion kind there:
+ // - `None` to preserve at that width (e.g. a native vector atomic),
+ // - `Elementwise` to keep halving,
+ // - `CmpXChg` (or any other kind) to commit to that expansion for the
+ // current width.
+ Elementwise,
+
// Rewrite to a non-atomic form for use in a known non-preemptible
// environment.
NotAtomic
@@ -2495,10 +2508,19 @@ class LLVM_ABI TargetLoweringBase {
/// AtomicExpand pass.
virtual AtomicExpansionKind
shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const {
- if (RMWI->getOperation() == AtomicRMWInst::Xchg &&
- (RMWI->getValOperand()->getType()->isFloatingPointTy() ||
- RMWI->getValOperand()->getType()->isPointerTy()))
- return AtomicExpansionKind::CastToInteger;
+ if (RMWI->getOperation() == AtomicRMWInst::Xchg) {
+ Type *Ty = RMWI->getValOperand()->getType();
+ if (Ty->isFloatingPointTy() || Ty->isPointerTy())
+ return AtomicExpansionKind::CastToInteger;
+
+ if (Ty->isVectorTy()) {
+ TypeSize SizeInBits =
+ RMWI->getDataLayout().getTypeStoreSizeInBits(Ty);
+ if (!SizeInBits.isScalable() &&
+ SizeInBits.getFixedValue() <= getMaxAtomicSizeInBitsSupported())
+ return AtomicExpansionKind::CastToInteger;
+ }
+ }
return AtomicExpansionKind::None;
}
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 160382168489f..970d10bb1e59b 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -44,6 +44,7 @@
#include "llvm/IR/Value.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
+#include "llvm/Support/Alignment.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
@@ -105,6 +106,7 @@ class AtomicExpandImpl {
bool tryExpandAtomicStore(StoreInst *SI);
void expandAtomicStoreToXChg(StoreInst *SI);
bool tryExpandAtomicRMW(AtomicRMWInst *AI);
+ void expandElementwiseAtomicRMW(AtomicRMWInst *AI);
AtomicRMWInst *convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI);
Value *
insertRMWLLSCLoop(IRBuilderBase &Builder, Type *ResultTy, Value *Addr,
@@ -380,6 +382,24 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
}
if (auto *RMWI = dyn_cast<AtomicRMWInst>(I)) {
+ if (RMWI->isElementwise()) {
+ auto ExpansionKind = TLI->shouldExpandAtomicRMWInIR(RMWI);
+ if (ExpansionKind ==
+ TargetLoweringBase::AtomicExpansionKind::Elementwise) {
+ // If the target wants fence-based lowering for this elementwise RMW, insert
+ // the fences before splitting and downgrade the RMW ordering first.
+ // The split operations inherit the downgraded ordering, so they do not each
+ // get their own fence pair.
+ tryInsertFencesForAtomic(
+ RMWI,
+ isReleaseOrStronger(RMWI->getOrdering()) ||
+ isAcquireOrStronger(RMWI->getOrdering()),
+ TLI->atomicOperationOrderAfterFenceSplit(RMWI));
+ expandElementwiseAtomicRMW(RMWI);
+ return true;
+ }
+ }
+
if (!atomicSizeSupported(TLI, RMWI)) {
expandAtomicRMWToLibcall(RMWI);
return true;
@@ -604,6 +624,124 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
return NewRMWI;
}
+/// Halve an elementwise vector atomicrmw and feed each half back through the
+/// normal atomic expansion pipeline. The target's shouldExpandAtomicRMWInIR()
+/// is re-queried at the halved width and may return any expansion kind there
+/// (e.g. None to preserve as a native vector atomic, Elementwise to keep
+/// halving, CmpXChg to emit one wide cmpxchg loop at the halved width). Three
+/// cases:
+/// N == 1: collapse directly to one scalar atomicrmw T
+/// N == 2: split into two scalar atomicrmw T (low at Ptr, high at
+/// gep inbounds T, Ptr, 1); reassemble via two insertelement's.
+/// N > 2: split into two atomicrmw elementwise <N/2 x T> (low at Ptr, high
+/// at gep inbounds <N/2 x T>, Ptr, 1); reassemble via a
+/// shufflevector.
+/// Note that halving works because the vectors must always be a power of two.
+///
+/// Each new atomicrmw inherits the original ordering/syncscope/volatility and
+/// metadata, and is fed back through processAtomicInstr() so further expansion fires per-half.
+void AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
+ assert(AI->isElementwise());
+ auto *VecTy = cast<FixedVectorType>(AI->getType());
+ Type *LaneTy = VecTy->getElementType();
+ const unsigned NumLanes = VecTy->getNumElements();
+ Value *Ptr = AI->getPointerOperand();
+ Value *Val = AI->getValOperand();
+
+ ReplacementIRBuilder Builder(AI, *DL);
+ IntegerType *IdxTy =
+ DL->getIndexType(AI->getContext(), AI->getPointerAddressSpace());
+
+ auto CreateRMWInstruction = [&](Value *HalfPtr, Value *HalfVal, Align A,
+ bool Elementwise) -> AtomicRMWInst * {
+ auto *NewAI =
+ Builder.CreateAtomicRMW(AI->getOperation(), HalfPtr, HalfVal, A,
+ AI->getOrdering(), AI->getSyncScopeID());
+ NewAI->setVolatile(AI->isVolatile());
+ NewAI->setElementwise(Elementwise);
+ copyMetadataForAtomic(*NewAI, *AI);
+ return NewAI;
+ };
+
+ Constant *IdxZero = ConstantInt::get(IdxTy, 0);
+ Constant *IdxOne = ConstantInt::get(IdxTy, 1);
+
+ // N == 1: collapse to a single scalar atomicrmw T and wrap the result back
+ // into a <1 x T> vector.
+ if (NumLanes == 1) {
+ Value *LaneVal = Builder.CreateExtractElement(Val, IdxZero, "lane.val");
+ AtomicRMWInst *LaneRMW = CreateRMWInstruction(Ptr, LaneVal, AI->getAlign(),
+ /*Elementwise=*/false);
+ Value *Result = Builder.CreateInsertElement(PoisonValue::get(VecTy),
+ LaneRMW, IdxZero, "lane.old");
+ AI->replaceAllUsesWith(Result);
+ AI->eraseFromParent();
+ processAtomicInstr(LaneRMW);
+ return;
+ }
+
+ const uint64_t LaneBytes = DL->getTypeStoreSize(LaneTy).getFixedValue();
+
+ // N == 2 base case: split directly into two scalar atomicrmw T, one per
+ // lane.
+ if (NumLanes == 2) {
+ Value *LoVal = Builder.CreateExtractElement(Val, IdxZero, "lo.val");
+ Value *HiVal = Builder.CreateExtractElement(Val, IdxOne, "hi.val");
+ Value *HiPtr = Builder.CreateInBoundsGEP(LaneTy, Ptr, IdxOne, "hi.ptr");
+ Align LoAlign = AI->getAlign();
+ Align HiAlign = commonAlignment(LoAlign, LaneBytes);
+
+ AtomicRMWInst *LoRMW =
+ CreateRMWInstruction(Ptr, LoVal, LoAlign, /*Elementwise=*/false);
+ AtomicRMWInst *HiRMW =
+ CreateRMWInstruction(HiPtr, HiVal, HiAlign, /*Elementwise=*/false);
+
+ Value *Result = PoisonValue::get(VecTy);
+ Result = Builder.CreateInsertElement(Result, LoRMW, IdxZero, "lo.old");
+ Result = Builder.CreateInsertElement(Result, HiRMW, IdxOne, "hi.old");
+ AI->replaceAllUsesWith(Result);
+ AI->eraseFromParent();
+ processAtomicInstr(LoRMW);
+ processAtomicInstr(HiRMW);
+ return;
+ }
+
+ // N > 2: split into two <N/2 x T> elementwise atomicrmws and recurse via
+ // processAtomicInstr().
+ assert(NumLanes % 2 == 0 &&
+ "elementwise atomicrmw vector length must be a power of two");
+ const unsigned HalfLanes = NumLanes / 2;
+ auto *HalfVecTy = FixedVectorType::get(LaneTy, HalfLanes);
+ const uint64_t HalfBytes = DL->getTypeStoreSize(HalfVecTy).getFixedValue();
+
+ SmallVector<int, 16> LoMask(HalfLanes), HiMask(HalfLanes);
+ for (unsigned I = 0; I < HalfLanes; ++I) {
+ LoMask[I] = static_cast<int>(I);
+ HiMask[I] = static_cast<int>(HalfLanes + I);
+ }
+ Value *LoVal = Builder.CreateShuffleVector(Val, LoMask, "lo.val");
+ Value *HiVal = Builder.CreateShuffleVector(Val, HiMask, "hi.val");
+
+ Value *HiPtr = Builder.CreateInBoundsGEP(HalfVecTy, Ptr, IdxOne, "hi.ptr");
+ Align LoAlign = AI->getAlign();
+ Align HiAlign = commonAlignment(LoAlign, HalfBytes);
+
+ AtomicRMWInst *LoRMW =
+ CreateRMWInstruction(Ptr, LoVal, LoAlign, /*Elementwise=*/true);
+ AtomicRMWInst *HiRMW =
+ CreateRMWInstruction(HiPtr, HiVal, HiAlign, /*Elementwise=*/true);
+
+ SmallVector<int, 16> Mask(NumLanes);
+ for (unsigned I = 0; I < NumLanes; ++I)
+ Mask[I] = static_cast<int>(I);
+ Value *Result = Builder.CreateShuffleVector(LoRMW, HiRMW, Mask, "old");
+
+ AI->replaceAllUsesWith(Result);
+ AI->eraseFromParent();
+ processAtomicInstr(LoRMW);
+ processAtomicInstr(HiRMW);
+}
+
bool AtomicExpandImpl::tryExpandAtomicLoad(LoadInst *LI) {
switch (TLI->shouldExpandAtomicLoadInIR(LI)) {
case TargetLoweringBase::AtomicExpansionKind::None:
@@ -1018,6 +1156,13 @@ static Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op,
case AtomicRMWInst::Add:
case AtomicRMWInst::Sub:
case AtomicRMWInst::Nand: {
+ if (PMV.ValueType->isVectorTy()) {
+ // Avoid letting packed integer arithmetic carry across vector lanes.
+ Value *Loaded_Extract = extractMaskedValue(Builder, Loaded, PMV);
+ Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded_Extract, Inc);
+ return insertMaskedValue(Builder, Loaded, NewVal, PMV);
+ }
+
// The other arithmetic ops need to be masked into place.
Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded, Shifted_Inc);
Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask);
@@ -1080,8 +1225,10 @@ void AtomicExpandImpl::expandPartwordAtomicRMW(
AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
Value *ValOperand_Shifted = nullptr;
- if (Op == AtomicRMWInst::Xchg || Op == AtomicRMWInst::Add ||
- Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Nand) {
+ if (Op == AtomicRMWInst::Xchg ||
+ (!PMV.ValueType->isVectorTy() &&
+ (Op == AtomicRMWInst::Add || Op == AtomicRMWInst::Sub ||
+ Op == AtomicRMWInst::Nand))) {
Value *ValOp = Builder.CreateBitCast(AI->getValOperand(), PMV.IntValueType);
ValOperand_Shifted =
Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
@@ -1124,9 +1271,15 @@ AtomicRMWInst *AtomicExpandImpl::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
- Value *ValOperand_Shifted =
- Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
- PMV.ShiftAmt, "ValOperand_Shifted");
+ Value *ValOp = AI->getValOperand();
+ if (ValOp->getType()->isVectorTy())
+ // For vectors, bitcast to the integer type before extending. Note that
+ // or/xor/and on vectors are equivalent to the same operation on an integer
+ // that spans the vector, so we can use the integer type for the operation.
+ ValOp = Builder.CreateBitCast(ValOp, PMV.IntValueType);
+ Value *ValOperand_Shifted = Builder.CreateShl(
+ Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
+ "ValOperand_Shifted");
Value *NewOperand;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index edd372a9db344..59842422f0168 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7456,12 +7456,15 @@ void NVPTXTargetLowering::ReplaceNodeResults(
}
}
-NVPTXTargetLowering::AtomicExpansionKind
-NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
- Type *Ty = AI->getValOperand()->getType();
+/// Returns how NVPTX should expand a scalar atomicrmw with the given op and
+/// value type.
+static TargetLoweringBase::AtomicExpansionKind
+getScalarAtomicRMWExpansion(AtomicRMWInst::BinOp Op, Type *Ty,
+ const NVPTXSubtarget &STI) {
+ using AtomicExpansionKind = TargetLoweringBase::AtomicExpansionKind;
- if (AI->isFloatingPointOperation()) {
- if (AI->getOperation() == AtomicRMWInst::BinOp::FAdd) {
+ if (AtomicRMWInst::isFPOperation(Op)) {
+ if (Op == AtomicRMWInst::FAdd) {
if (Ty->isHalfTy() && STI.getSmVersion() >= 70 &&
STI.getPTXVersion() >= 63)
return AtomicExpansionKind::None;
@@ -7473,22 +7476,24 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
if (Ty->isDoubleTy() && STI.hasAtomAddF64())
return AtomicExpansionKind::None;
}
+ // Scalar atom.{min,max,sub}.{f32,f64} and related FP ops are not
+ // supported by the PTX ISA; use cmpxchg emulation for them.
return AtomicExpansionKind::CmpXChg;
}
- assert(Ty->isIntegerTy() && "Ty should be integer at this point");
+ assert(Ty->isIntegerTy() && "Ty should be integer at this point. Integer operations must act on an integer operand. We already cast non-integer CmpXChg operands to integer.");
const unsigned BitWidth = cast<IntegerType>(Ty)->getBitWidth();
- switch (AI->getOperation()) {
+ switch (Op) {
default:
return AtomicExpansionKind::CmpXChg;
- case AtomicRMWInst::BinOp::Xchg:
+ case AtomicRMWInst::Xchg:
if (BitWidth == 128)
return AtomicExpansionKind::None;
[[fallthrough]];
- case AtomicRMWInst::BinOp::And:
- case AtomicRMWInst::BinOp::Or:
- case AtomicRMWInst::BinOp::Xor:
+ case AtomicRMWInst::And:
+ case AtomicRMWInst::Or:
+ case AtomicRMWInst::Xor:
switch (BitWidth) {
case 8:
case 16:
@@ -7504,12 +7509,12 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
default:
llvm_unreachable("unsupported width encountered");
}
- case AtomicRMWInst::BinOp::Add:
- case AtomicRMWInst::BinOp::Sub:
- case AtomicRMWInst::BinOp::Max:
- case AtomicRMWInst::BinOp::Min:
- case AtomicRMWInst::BinOp::UMax:
- case AtomicRMWInst::BinOp::UMin:
+ case AtomicRMWInst::Add:
+ case AtomicRMWInst::Sub:
+ case AtomicRMWInst::Max:
+ case AtomicRMWInst::Min:
+ case AtomicRMWInst::UMax:
+ case AtomicRMWInst::UMin:
switch (BitWidth) {
case 8:
case 16:
@@ -7525,8 +7530,8 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
default:
llvm_unreachable("unsupported width encountered");
}
- case AtomicRMWInst::BinOp::UIncWrap:
- case AtomicRMWInst::BinOp::UDecWrap:
+ case AtomicRMWInst::UIncWrap:
+ case AtomicRMWInst::UDecWrap:
switch (BitWidth) {
case 32:
return AtomicExpansionKind::None;
@@ -7543,6 +7548,38 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
return AtomicExpansionKind::CmpXChg;
}
+NVPTXTargetLowering::AtomicExpansionKind
+NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
+ // TODO: once we support native elementwise vector atoms,
+ // return `AtomicExpansionKind::None` to preserve and lower them.
+ if (AI->isElementwise()) {
+ auto *VecTy = cast<FixedVectorType>(AI->getType());
+ Type *LaneTy = VecTy->getElementType();
+
+ // If the scalar lane op is natively supported, return
+ // Elementwise so halving eventually bottoms out at the scalar base
+ // case, where this hook returns None for each scalar lane and the
+ // scalar `atom.*` instruction is preserved.
+ if (getScalarAtomicRMWExpansion(AI->getOperation(), LaneTy, STI) ==
+ AtomicExpansionKind::None)
+ return AtomicExpansionKind::Elementwise;
+
+ // If the whole vector fits a single native cmpxchg, emit one wide
+ // cmpxchg loop at the current width.
+ const DataLayout &DL = AI->getDataLayout();
+ uint64_t VecBits = DL.getTypeStoreSizeInBits(VecTy).getFixedValue();
+ if (VecBits <= getMaxAtomicSizeInBitsSupported())
+ return AtomicExpansionKind::CmpXChg;
+
+ // If the vector is too wide for a single native cmpxchg, halve further to
+ // emit multiple cmpxchg loops.
+ return AtomicExpansionKind::Elementwise;
+ }
+
+ return getScalarAtomicRMWExpansion(AI->getOperation(),
+ AI->getValOperand()->getType(), STI);
+}
+
bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
const Instruction *I) const {
// This function returns true iff the operation is emulated using a CAS-loop,
@@ -7564,9 +7601,12 @@ bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
return (cast<IntegerType>(CI->getCompareOperand()->getType())
->getBitWidth() < STI.getMinCmpXchgSizeInBits()) ||
CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent;
- if (auto *RI = dyn_cast<AtomicRMWInst>(I))
- return shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::CmpXChg ||
+ if (auto *RI = dyn_cast<AtomicRMWInst>(I)) {
+ AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(RI);
+ return Expansion == AtomicExpansionKind::CmpXChg ||
+ Expansion == AtomicExpansionKind::Elementwise ||
RI->getOrdering() == AtomicOrdering::SequentiallyConsistent;
+ }
return false;
}
@@ -7642,10 +7682,15 @@ Instruction *NVPTXTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
auto SSID = getAtomicSyncScopeID(Inst);
assert(SSID.has_value() && "Expected an atomic operation");
- bool IsEmulated =
- CI ? cast<IntegerType>(CI->getCompareOperand()->getType())
- ->getBitWidth() < STI.getMinCmpXchgSizeInBits()
- : shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::CmpXChg;
+ bool IsEmulated = false;
+ if (CI)
+ IsEmulated = cast<IntegerType>(CI->getCompareOperand()->getType())
+ ->getBitWidth() < STI.getMinCmpXchgSizeInBits();
+ else {
+ AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(RI);
+ IsEmulated = Expansion == AtomicExpansionKind::CmpXChg ||
+ Expansion == AtomicExpansionKind::Elementwise;
+ }
if (isAcquireOrStronger(Ord) && IsEmulated)
return Builder.CreateFence(AtomicOrdering::Acquire, SSID.value());
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index f2879c87dafc7..af40de10b9430 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -147,6 +147,17 @@ class NVPTXTargetLowering : public TargetLowering {
bool isCheapToSpeculateCtlz(Type *Ty) const override { return true; }
AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const override {
+ Type *Ty = LI->getType();
+ // SelectionDAG can split vector computations, but not vector atomic loads.
+ // Splitting those loads into per-lane atomics would change the atomicity,
+ // so cast them to a single same-width integer atomic load instead.
+ if (Ty->isVectorTy()) {
+ TypeSize SizeInBits = LI->getDataLayout().getTypeStoreSizeInBits(Ty);
+ if (!SizeInBits.isScalable() &&
+ SizeInBits.getFixedValue() <= getMaxAtomicSizeInBitsSupported())
+ return AtomicExpansionKind::CastToInteger;
+ }
+
return AtomicExpansionKind::None;
}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
new file mode 100644
index 0000000000000..bb279c9a92e5f
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
@@ -0,0 +1,16128 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | FileCheck %s --check-prefix=SM60
+; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | %ptxas-verify -arch=sm_60 %}
+
+define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: xchg_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<14>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
+; SM60-NEXT: mov.b32 %r8, 65535;
+; SM60-NEXT: shl.b32 %r9, %r8, %r1;
+; SM60-NEXT: not.b32 %r2, %r9;
+; SM60-NEXT: shl.b32 %r3, %r5, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r13, [%rd1];
+; SM60-NEXT: $L__BB0_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r13, %r2;
+; SM60-NEXT: or.b32 %r11, %r10, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM60-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM60-NEXT: mov.b32 %r13, %r4;
+; SM60-NEXT: @%p1 bra $L__BB0_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r12, %r4, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r12;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @xchg_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: xchg_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @xchg_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: xchg_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @xchg_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: xchg_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @xchg_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: xchg_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @xchg_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: xchg_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: cvt.u64.u32 %rd2, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r2;
+; SM60-NEXT: shl.b64 %rd4, %rd3, 32;
+; SM60-NEXT: or.b64 %rd5, %rd2, %rd4;
+; SM60-NEXT: cvt.u64.u32 %rd6, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: shl.b64 %rd8, %rd7, 32;
+; SM60-NEXT: or.b64 %rd9, %rd6, %rd8;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd10, [%rd1], %rd5;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @xchg_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: xchg_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [xchg_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.exch.b32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @xchg_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: xchg_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.exch.b32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.exch.b32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.exch.b32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @xchg_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: xchg_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xchg_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.exch.b32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.exch.b32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.exch.b32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.exch.b32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.exch.b32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.exch.b32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.exch.b32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @xchg_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: xchg_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @xchg_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: xchg_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @xchg_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: xchg_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xchg_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xchg_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB12_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB12_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: add_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: $L__BB13_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB13_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: add_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB14_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM60-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM60-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM60-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB14_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: add_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB15_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: add.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: add.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB15_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: add_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB16_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: add.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: add.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM60-NEXT: add.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: add.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB16_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: add_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB17_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: add.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: add.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM60-NEXT: add.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: add.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB17_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: $L__BB17_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM60-NEXT: add.s16 %rs17, %rs16, %rs14;
+; SM60-NEXT: add.s16 %rs18, %rs15, %rs13;
+; SM60-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: add.s16 %rs23, %rs22, %rs20;
+; SM60-NEXT: add.s16 %rs24, %rs21, %rs19;
+; SM60-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB17_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @add_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: add_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: add_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @add_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: add_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @add_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: add_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.add.u64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.add.u64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @add_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: add_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [add_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [add_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.add.u64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.add.u64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.add.u64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.add.u64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.add.u64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.add.u64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: sub_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [sub_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB24_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: sub.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: sub.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB24_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: sub_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [sub_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: $L__BB25_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: sub.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: sub.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB25_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: sub_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB26_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: sub.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: sub.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM60-NEXT: sub.s16 %rs15, %rs14, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM60-NEXT: sub.s16 %rs18, %rs17, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM60-NEXT: sub.s16 %rs21, %rs20, %rs19;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM60-NEXT: sub.s16 %rs24, %rs23, %rs22;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB26_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: sub_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [sub_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB27_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB27_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: sub_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB28_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM60-NEXT: sub.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: sub.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB28_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: sub_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB29_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM60-NEXT: sub.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: sub.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB29_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: $L__BB29_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM60-NEXT: sub.s16 %rs17, %rs16, %rs14;
+; SM60-NEXT: sub.s16 %rs18, %rs15, %rs13;
+; SM60-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: sub.s16 %rs23, %rs22, %rs20;
+; SM60-NEXT: sub.s16 %rs24, %rs21, %rs19;
+; SM60-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB29_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: sub_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: neg.s32 %r3, %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1], %r3;
+; SM60-NEXT: neg.s32 %r5, %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r5;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @sub_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: sub_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: neg.s32 %r5, %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1], %r5;
+; SM60-NEXT: neg.s32 %r7, %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+4], %r7;
+; SM60-NEXT: neg.s32 %r9, %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+8], %r9;
+; SM60-NEXT: neg.s32 %r11, %r4;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r11;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: sub_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: neg.s32 %r9, %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1], %r9;
+; SM60-NEXT: neg.s32 %r11, %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+4], %r11;
+; SM60-NEXT: neg.s32 %r13, %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+8], %r13;
+; SM60-NEXT: neg.s32 %r15, %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+12], %r15;
+; SM60-NEXT: neg.s32 %r17, %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r18, [%rd1+16], %r17;
+; SM60-NEXT: neg.s32 %r19, %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r20, [%rd1+20], %r19;
+; SM60-NEXT: neg.s32 %r21, %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r22, [%rd1+24], %r21;
+; SM60-NEXT: neg.s32 %r23, %r4;
+; SM60-NEXT: atom.cta.global.add.u32 %r24, [%rd1+28], %r23;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: sub_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<8>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: neg.s64 %rd4, %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd5, [%rd1], %rd4;
+; SM60-NEXT: neg.s64 %rd6, %rd3;
+; SM60-NEXT: atom.cta.global.add.u64 %rd7, [%rd1+8], %rd6;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @sub_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: sub_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<14>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: neg.s64 %rd6, %rd4;
+; SM60-NEXT: atom.cta.global.add.u64 %rd7, [%rd1], %rd6;
+; SM60-NEXT: neg.s64 %rd8, %rd5;
+; SM60-NEXT: atom.cta.global.add.u64 %rd9, [%rd1+8], %rd8;
+; SM60-NEXT: neg.s64 %rd10, %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd11, [%rd1+16], %rd10;
+; SM60-NEXT: neg.s64 %rd12, %rd3;
+; SM60-NEXT: atom.cta.global.add.u64 %rd13, [%rd1+24], %rd12;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd11, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd7, %rd9};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: sub_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<26>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [sub_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: neg.s64 %rd10, %rd8;
+; SM60-NEXT: atom.cta.global.add.u64 %rd11, [%rd1], %rd10;
+; SM60-NEXT: neg.s64 %rd12, %rd9;
+; SM60-NEXT: atom.cta.global.add.u64 %rd13, [%rd1+8], %rd12;
+; SM60-NEXT: neg.s64 %rd14, %rd6;
+; SM60-NEXT: atom.cta.global.add.u64 %rd15, [%rd1+16], %rd14;
+; SM60-NEXT: neg.s64 %rd16, %rd7;
+; SM60-NEXT: atom.cta.global.add.u64 %rd17, [%rd1+24], %rd16;
+; SM60-NEXT: neg.s64 %rd18, %rd4;
+; SM60-NEXT: atom.cta.global.add.u64 %rd19, [%rd1+32], %rd18;
+; SM60-NEXT: neg.s64 %rd20, %rd5;
+; SM60-NEXT: atom.cta.global.add.u64 %rd21, [%rd1+40], %rd20;
+; SM60-NEXT: neg.s64 %rd22, %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd23, [%rd1+48], %rd22;
+; SM60-NEXT: neg.s64 %rd24, %rd3;
+; SM60-NEXT: atom.cta.global.add.u64 %rd25, [%rd1+56], %rd24;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd23, %rd25};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd19, %rd21};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd15, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd11, %rd13};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @and_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: and_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<12>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM60-NEXT: and.b32 %r2, %r1, 3;
+; SM60-NEXT: shl.b32 %r3, %r2, 3;
+; SM60-NEXT: mov.b32 %r4, 65535;
+; SM60-NEXT: shl.b32 %r5, %r4, %r3;
+; SM60-NEXT: not.b32 %r6, %r5;
+; SM60-NEXT: ld.param.b16 %r7, [and_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r8, %r7, %r3;
+; SM60-NEXT: or.b32 %r9, %r8, %r6;
+; SM60-NEXT: atom.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM60-NEXT: shr.u32 %r11, %r10, %r3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r11;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @and_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: and_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [and_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB37_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB37_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: and_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB38_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: and.b32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB38_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: and_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [and_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB39_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB39_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: and_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB40_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: and.b32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB40_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: and_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB41_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: and.b32 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB41_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB41_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: and.b32 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB41_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @and_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: and_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.and.b32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @and_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: and_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.and.b32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.and.b32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.and.b32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @and_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: and_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [and_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.and.b32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.and.b32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.and.b32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.and.b32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.and.b32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.and.b32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.and.b32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @and_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: and_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.and.b64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @and_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: and_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.and.b64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.and.b64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.and.b64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @and_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: and_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [and_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [and_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.and.b64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.and.b64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.and.b64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.and.b64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.and.b64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.and.b64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.and.b64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: nand_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: $L__BB48_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r18, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, %r4;
+; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM60-NEXT: shl.b32 %r14, %r13, %r1;
+; SM60-NEXT: and.b32 %r15, %r18, %r2;
+; SM60-NEXT: or.b32 %r16, %r15, %r14;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: @%p1 bra $L__BB48_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r17, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB49_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB49_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB50_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB50_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: nand_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB51_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB51_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: nand_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB52_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB52_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: nand_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM60-NEXT: $L__BB53_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r14, %r2;
+; SM60-NEXT: and.b32 %r6, %r13, %r1;
+; SM60-NEXT: xor.b32 %r7, %r6, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM60-NEXT: xor.b32 %r8, %r5, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB53_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM60-NEXT: $L__BB53_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r15;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r16;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: and.b32 %r9, %r16, %r4;
+; SM60-NEXT: and.b32 %r10, %r15, %r3;
+; SM60-NEXT: xor.b32 %r11, %r10, -1;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: xor.b32 %r12, %r9, -1;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB53_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB54_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB54_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB55_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r10, %r2;
+; SM60-NEXT: and.b32 %r6, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB55_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM60-NEXT: $L__BB55_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r7, %r12, %r4;
+; SM60-NEXT: and.b32 %r8, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB55_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: nand_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<46>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB56_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r9, %r18, %r6;
+; SM60-NEXT: and.b32 %r10, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB56_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM60-NEXT: $L__BB56_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r11, %r20, %r8;
+; SM60-NEXT: and.b32 %r12, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB56_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM60-NEXT: $L__BB56_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM60-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM60-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM60-NEXT: and.b32 %r13, %r22, %r2;
+; SM60-NEXT: and.b32 %r14, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM60-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM60-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM60-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM60-NEXT: not.b64 %rd33, %rd32;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM60-NEXT: @%p3 bra $L__BB56_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM60-NEXT: $L__BB56_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM60-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM60-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM60-NEXT: and.b32 %r15, %r24, %r4;
+; SM60-NEXT: and.b32 %r16, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM60-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM60-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM60-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM60-NEXT: not.b64 %rd44, %rd43;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM60-NEXT: @%p4 bra $L__BB56_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: nand_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [nand_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
+; SM60-NEXT: $L__BB57_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd6, %rd10, %rd1;
+; SM60-NEXT: not.b64 %rd7, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
+; SM60-NEXT: mov.b64 %rd10, %rd3;
+; SM60-NEXT: @%p1 bra $L__BB57_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
+; SM60-NEXT: $L__BB57_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd8, %rd11, %rd2;
+; SM60-NEXT: not.b64 %rd9, %rd8;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
+; SM60-NEXT: mov.b64 %rd11, %rd4;
+; SM60-NEXT: @%p2 bra $L__BB57_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: nand_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [nand_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
+; SM60-NEXT: $L__BB58_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd18;
+; SM60-NEXT: and.b64 %rd10, %rd5, %rd3;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
+; SM60-NEXT: @%p1 bra $L__BB58_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
+; SM60-NEXT: $L__BB58_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd19;
+; SM60-NEXT: and.b64 %rd12, %rd6, %rd4;
+; SM60-NEXT: not.b64 %rd13, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM60-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
+; SM60-NEXT: @%p2 bra $L__BB58_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
+; SM60-NEXT: $L__BB58_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd14, %rd20, %rd1;
+; SM60-NEXT: not.b64 %rd15, %rd14;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
+; SM60-NEXT: mov.b64 %rd20, %rd7;
+; SM60-NEXT: @%p3 bra $L__BB58_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
+; SM60-NEXT: $L__BB58_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd16, %rd21, %rd2;
+; SM60-NEXT: not.b64 %rd17, %rd16;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
+; SM60-NEXT: mov.b64 %rd21, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB58_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: nand_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [nand_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [nand_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [nand_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
+; SM60-NEXT: $L__BB59_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd34;
+; SM60-NEXT: and.b64 %rd18, %rd7, %rd5;
+; SM60-NEXT: not.b64 %rd19, %rd18;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM60-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
+; SM60-NEXT: @%p1 bra $L__BB59_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
+; SM60-NEXT: $L__BB59_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd35;
+; SM60-NEXT: and.b64 %rd20, %rd8, %rd6;
+; SM60-NEXT: not.b64 %rd21, %rd20;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM60-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
+; SM60-NEXT: @%p2 bra $L__BB59_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
+; SM60-NEXT: $L__BB59_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd36;
+; SM60-NEXT: and.b64 %rd22, %rd9, %rd3;
+; SM60-NEXT: not.b64 %rd23, %rd22;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM60-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
+; SM60-NEXT: @%p3 bra $L__BB59_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
+; SM60-NEXT: $L__BB59_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd37;
+; SM60-NEXT: and.b64 %rd24, %rd10, %rd4;
+; SM60-NEXT: not.b64 %rd25, %rd24;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM60-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
+; SM60-NEXT: @%p4 bra $L__BB59_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
+; SM60-NEXT: $L__BB59_9: // %atomicrmw.start41
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd38;
+; SM60-NEXT: and.b64 %rd26, %rd13, %rd1;
+; SM60-NEXT: not.b64 %rd27, %rd26;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM60-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
+; SM60-NEXT: @%p5 bra $L__BB59_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end40
+; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
+; SM60-NEXT: $L__BB59_11: // %atomicrmw.start47
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd39;
+; SM60-NEXT: and.b64 %rd28, %rd14, %rd2;
+; SM60-NEXT: not.b64 %rd29, %rd28;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM60-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
+; SM60-NEXT: @%p6 bra $L__BB59_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end46
+; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
+; SM60-NEXT: $L__BB59_13: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd30, %rd40, %rd11;
+; SM60-NEXT: not.b64 %rd31, %rd30;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
+; SM60-NEXT: mov.b64 %rd40, %rd15;
+; SM60-NEXT: @%p7 bra $L__BB59_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end57
+; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
+; SM60-NEXT: $L__BB59_15: // %atomicrmw.start64
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd32, %rd41, %rd12;
+; SM60-NEXT: not.b64 %rd33, %rd32;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
+; SM60-NEXT: mov.b64 %rd41, %rd16;
+; SM60-NEXT: @%p8 bra $L__BB59_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end63
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @or_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: or_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM60-NEXT: and.b32 %r2, %r1, 3;
+; SM60-NEXT: shl.b32 %r3, %r2, 3;
+; SM60-NEXT: ld.param.b16 %r4, [or_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r5, %r4, %r3;
+; SM60-NEXT: atom.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM60-NEXT: shr.u32 %r7, %r6, %r3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r7;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @or_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: or_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [or_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: or.b32 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB61_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: or_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: or.b32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: or.b32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB62_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: or_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [or_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: or.b32 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB63_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: or_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: or.b32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: or.b32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB64_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: or_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: or.b32 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: or.b32 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB65_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB65_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: or.b32 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: or.b32 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB65_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @or_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: or_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.or.b32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @or_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: or_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.or.b32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.or.b32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.or.b32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @or_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: or_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [or_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.or.b32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.or.b32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.or.b32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.or.b32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.or.b32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.or.b32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.or.b32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @or_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: or_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.or.b64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @or_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: or_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.or.b64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.or.b64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.or.b64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @or_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: or_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [or_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [or_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.or.b64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.or.b64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.or.b64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.or.b64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.or.b64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.or.b64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.or.b64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @xor_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: xor_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM60-NEXT: and.b32 %r2, %r1, 3;
+; SM60-NEXT: shl.b32 %r3, %r2, 3;
+; SM60-NEXT: ld.param.b16 %r4, [xor_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r5, %r4, %r3;
+; SM60-NEXT: atom.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM60-NEXT: shr.u32 %r7, %r6, %r3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r7;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @xor_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: xor_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [xor_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: xor.b32 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB73_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: xor_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: xor.b32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: xor.b32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB74_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: xor_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [xor_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: xor.b32 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB75_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: xor_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: xor.b32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: xor.b32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB76_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: xor_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: xor.b32 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: xor.b32 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB77_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB77_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: xor.b32 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: xor.b32 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB77_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @xor_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: xor_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.xor.b32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @xor_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: xor_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.xor.b32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.xor.b32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.xor.b32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @xor_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: xor_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xor_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.xor.b32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.xor.b32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.xor.b32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.xor.b32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.xor.b32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.xor.b32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.xor.b32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @xor_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: xor_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @xor_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: xor_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @xor_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: xor_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xor_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xor_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: max_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<14>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [max_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [max_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: cvt.s16.s8 %rs7, %rs5;
+; SM60-NEXT: cvt.s16.s8 %rs8, %rs6;
+; SM60-NEXT: $L__BB84_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r16, %r1;
+; SM60-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM60-NEXT: bfe.s32 %r10, %r9, 8, 8;
+; SM60-NEXT: cvt.s8.s32 %rs4, %r10;
+; SM60-NEXT: max.s16 %rs9, %rs4, %rs8;
+; SM60-NEXT: max.s16 %rs10, %rs3, %rs7;
+; SM60-NEXT: and.b16 %rs11, %rs10, 255;
+; SM60-NEXT: shl.b16 %rs12, %rs9, 8;
+; SM60-NEXT: or.b16 %rs13, %rs11, %rs12;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs13;
+; SM60-NEXT: shl.b32 %r12, %r11, %r1;
+; SM60-NEXT: and.b32 %r13, %r16, %r2;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r16;
+; SM60-NEXT: mov.b32 %r16, %r3;
+; SM60-NEXT: @%p1 bra $L__BB84_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: max_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b32 %r<27>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [max_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r26, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM60-NEXT: $L__BB85_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
+; SM60-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
+; SM60-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
+; SM60-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
+; SM60-NEXT: setp.gt.s32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r15, %r26, 0, 0x7773U;
+; SM60-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; SM60-NEXT: prmt.b32 %r17, %r26, 0, 0x7772U;
+; SM60-NEXT: selp.b32 %r18, %r17, %r13, %p3;
+; SM60-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
+; SM60-NEXT: prmt.b32 %r20, %r26, 0, 0x7771U;
+; SM60-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; SM60-NEXT: prmt.b32 %r22, %r26, 0, 0x7770U;
+; SM60-NEXT: selp.b32 %r23, %r22, %r11, %p1;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r26;
+; SM60-NEXT: mov.b32 %r26, %r1;
+; SM60-NEXT: @%p5 bra $L__BB85_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: max_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<51>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM60-NEXT: $L__BB86_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r49;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
+; SM60-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
+; SM60-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM60-NEXT: prmt.b32 %r8, %r50, 0, 0xaaa2U;
+; SM60-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM60-NEXT: prmt.b32 %r10, %r50, 0, 0xbbb3U;
+; SM60-NEXT: setp.gt.s32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r1, 0, 0x8880U;
+; SM60-NEXT: prmt.b32 %r12, %r49, 0, 0x8880U;
+; SM60-NEXT: setp.gt.s32 %p5, %r12, %r11;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x9991U;
+; SM60-NEXT: prmt.b32 %r14, %r49, 0, 0x9991U;
+; SM60-NEXT: setp.gt.s32 %p6, %r14, %r13;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0xaaa2U;
+; SM60-NEXT: prmt.b32 %r16, %r49, 0, 0xaaa2U;
+; SM60-NEXT: setp.gt.s32 %p7, %r16, %r15;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0xbbb3U;
+; SM60-NEXT: prmt.b32 %r18, %r49, 0, 0xbbb3U;
+; SM60-NEXT: setp.gt.s32 %p8, %r18, %r17;
+; SM60-NEXT: prmt.b32 %r19, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r19;
+; SM60-NEXT: prmt.b32 %r20, %r50, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r20;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r50, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r22;
+; SM60-NEXT: prmt.b32 %r23, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r23;
+; SM60-NEXT: prmt.b32 %r24, %r50, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r24;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r50, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r26;
+; SM60-NEXT: prmt.b32 %r27, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r27;
+; SM60-NEXT: prmt.b32 %r28, %r49, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r29;
+; SM60-NEXT: prmt.b32 %r30, %r49, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs12, %r30;
+; SM60-NEXT: prmt.b32 %r31, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r31;
+; SM60-NEXT: prmt.b32 %r32, %r49, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r32;
+; SM60-NEXT: prmt.b32 %r33, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r33;
+; SM60-NEXT: prmt.b32 %r34, %r49, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r34;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM60-NEXT: cvt.u32.u16 %r35, %rs17;
+; SM60-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs18;
+; SM60-NEXT: prmt.b32 %r37, %r36, %r35, 0x3340U;
+; SM60-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs19;
+; SM60-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM60-NEXT: cvt.u32.u16 %r39, %rs20;
+; SM60-NEXT: prmt.b32 %r40, %r39, %r38, 0x3340U;
+; SM60-NEXT: prmt.b32 %r41, %r40, %r37, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM60-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r42, %rs21;
+; SM60-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM60-NEXT: cvt.u32.u16 %r43, %rs22;
+; SM60-NEXT: prmt.b32 %r44, %r43, %r42, 0x3340U;
+; SM60-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM60-NEXT: cvt.u32.u16 %r45, %rs23;
+; SM60-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM60-NEXT: cvt.u32.u16 %r46, %rs24;
+; SM60-NEXT: prmt.b32 %r47, %r46, %r45, 0x3340U;
+; SM60-NEXT: prmt.b32 %r48, %r47, %r44, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r48;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB86_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: max_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [max_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB87_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: max.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: max.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB87_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: max_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB88_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: max.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: max.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM60-NEXT: max.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: max.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB88_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: max_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB89_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: max.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: max.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM60-NEXT: max.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: max.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB89_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: $L__BB89_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM60-NEXT: max.s16 %rs17, %rs16, %rs14;
+; SM60-NEXT: max.s16 %rs18, %rs15, %rs13;
+; SM60-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: max.s16 %rs23, %rs22, %rs20;
+; SM60-NEXT: max.s16 %rs24, %rs21, %rs19;
+; SM60-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB89_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @max_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: max_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.max.s32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @max_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: max_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.max.s32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.max.s32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.max.s32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @max_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: max_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [max_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.max.s32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.max.s32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.max.s32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.max.s32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.max.s32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.max.s32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.max.s32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @max_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: max_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.max.s64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @max_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: max_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.max.s64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.max.s64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.max.s64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @max_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: max_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [max_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [max_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.max.s64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.max.s64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.max.s64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.max.s64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.max.s64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.max.s64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.max.s64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: min_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<14>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [min_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [min_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: cvt.s16.s8 %rs7, %rs5;
+; SM60-NEXT: cvt.s16.s8 %rs8, %rs6;
+; SM60-NEXT: $L__BB96_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r16, %r1;
+; SM60-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM60-NEXT: bfe.s32 %r10, %r9, 8, 8;
+; SM60-NEXT: cvt.s8.s32 %rs4, %r10;
+; SM60-NEXT: min.s16 %rs9, %rs4, %rs8;
+; SM60-NEXT: min.s16 %rs10, %rs3, %rs7;
+; SM60-NEXT: and.b16 %rs11, %rs10, 255;
+; SM60-NEXT: shl.b16 %rs12, %rs9, 8;
+; SM60-NEXT: or.b16 %rs13, %rs11, %rs12;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs13;
+; SM60-NEXT: shl.b32 %r12, %r11, %r1;
+; SM60-NEXT: and.b32 %r13, %r16, %r2;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r16;
+; SM60-NEXT: mov.b32 %r16, %r3;
+; SM60-NEXT: @%p1 bra $L__BB96_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: min_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b32 %r<27>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [min_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r26, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM60-NEXT: $L__BB97_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
+; SM60-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
+; SM60-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
+; SM60-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
+; SM60-NEXT: setp.le.s32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r15, %r26, 0, 0x7773U;
+; SM60-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; SM60-NEXT: prmt.b32 %r17, %r26, 0, 0x7772U;
+; SM60-NEXT: selp.b32 %r18, %r17, %r13, %p3;
+; SM60-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
+; SM60-NEXT: prmt.b32 %r20, %r26, 0, 0x7771U;
+; SM60-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; SM60-NEXT: prmt.b32 %r22, %r26, 0, 0x7770U;
+; SM60-NEXT: selp.b32 %r23, %r22, %r11, %p1;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r26;
+; SM60-NEXT: mov.b32 %r26, %r1;
+; SM60-NEXT: @%p5 bra $L__BB97_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: min_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<51>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM60-NEXT: $L__BB98_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r49;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
+; SM60-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
+; SM60-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM60-NEXT: prmt.b32 %r8, %r50, 0, 0xaaa2U;
+; SM60-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM60-NEXT: prmt.b32 %r10, %r50, 0, 0xbbb3U;
+; SM60-NEXT: setp.le.s32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r1, 0, 0x8880U;
+; SM60-NEXT: prmt.b32 %r12, %r49, 0, 0x8880U;
+; SM60-NEXT: setp.le.s32 %p5, %r12, %r11;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x9991U;
+; SM60-NEXT: prmt.b32 %r14, %r49, 0, 0x9991U;
+; SM60-NEXT: setp.le.s32 %p6, %r14, %r13;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0xaaa2U;
+; SM60-NEXT: prmt.b32 %r16, %r49, 0, 0xaaa2U;
+; SM60-NEXT: setp.le.s32 %p7, %r16, %r15;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0xbbb3U;
+; SM60-NEXT: prmt.b32 %r18, %r49, 0, 0xbbb3U;
+; SM60-NEXT: setp.le.s32 %p8, %r18, %r17;
+; SM60-NEXT: prmt.b32 %r19, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r19;
+; SM60-NEXT: prmt.b32 %r20, %r50, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r20;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r50, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r22;
+; SM60-NEXT: prmt.b32 %r23, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r23;
+; SM60-NEXT: prmt.b32 %r24, %r50, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r24;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r50, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r26;
+; SM60-NEXT: prmt.b32 %r27, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r27;
+; SM60-NEXT: prmt.b32 %r28, %r49, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r29;
+; SM60-NEXT: prmt.b32 %r30, %r49, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs12, %r30;
+; SM60-NEXT: prmt.b32 %r31, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r31;
+; SM60-NEXT: prmt.b32 %r32, %r49, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r32;
+; SM60-NEXT: prmt.b32 %r33, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r33;
+; SM60-NEXT: prmt.b32 %r34, %r49, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r34;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM60-NEXT: cvt.u32.u16 %r35, %rs17;
+; SM60-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs18;
+; SM60-NEXT: prmt.b32 %r37, %r36, %r35, 0x3340U;
+; SM60-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs19;
+; SM60-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM60-NEXT: cvt.u32.u16 %r39, %rs20;
+; SM60-NEXT: prmt.b32 %r40, %r39, %r38, 0x3340U;
+; SM60-NEXT: prmt.b32 %r41, %r40, %r37, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM60-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r42, %rs21;
+; SM60-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM60-NEXT: cvt.u32.u16 %r43, %rs22;
+; SM60-NEXT: prmt.b32 %r44, %r43, %r42, 0x3340U;
+; SM60-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM60-NEXT: cvt.u32.u16 %r45, %rs23;
+; SM60-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM60-NEXT: cvt.u32.u16 %r46, %rs24;
+; SM60-NEXT: prmt.b32 %r47, %r46, %r45, 0x3340U;
+; SM60-NEXT: prmt.b32 %r48, %r47, %r44, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r48;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB98_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: min_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [min_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB99_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: min.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: min.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB99_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: min_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB100_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: min.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: min.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM60-NEXT: min.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: min.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB100_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: min_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB101_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: min.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: min.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM60-NEXT: min.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: min.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB101_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: $L__BB101_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM60-NEXT: min.s16 %rs17, %rs16, %rs14;
+; SM60-NEXT: min.s16 %rs18, %rs15, %rs13;
+; SM60-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: min.s16 %rs23, %rs22, %rs20;
+; SM60-NEXT: min.s16 %rs24, %rs21, %rs19;
+; SM60-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB101_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @min_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: min_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.min.s32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @min_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: min_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.min.s32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.min.s32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.min.s32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @min_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: min_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [min_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.min.s32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.min.s32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.min.s32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.min.s32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.min.s32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.min.s32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.min.s32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @min_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: min_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.min.s64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @min_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: min_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.min.s64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.min.s64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.min.s64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @min_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: min_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [min_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [min_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.min.s64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.min.s64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.min.s64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.min.s64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.min.s64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.min.s64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.min.s64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: umax_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<18>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umax_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM60-NEXT: $L__BB108_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r17, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM60-NEXT: max.u16 %rs9, %rs5, %rs7;
+; SM60-NEXT: max.u16 %rs10, %rs6, %rs8;
+; SM60-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM60-NEXT: or.b16 %rs12, %rs9, %rs11;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM60-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NEXT: mov.b32 %r17, %r3;
+; SM60-NEXT: @%p1 bra $L__BB108_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: umax_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [umax_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM60-NEXT: $L__BB109_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
+; SM60-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
+; SM60-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
+; SM60-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
+; SM60-NEXT: setp.gt.u32 %p4, %r10, %r9;
+; SM60-NEXT: selp.b32 %r11, %r10, %r9, %p4;
+; SM60-NEXT: selp.b32 %r12, %r8, %r7, %p3;
+; SM60-NEXT: prmt.b32 %r13, %r12, %r11, 0x3340U;
+; SM60-NEXT: selp.b32 %r14, %r6, %r5, %p2;
+; SM60-NEXT: selp.b32 %r15, %r4, %r3, %p1;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r14, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r13, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p5 bra $L__BB109_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: umax_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM60-NEXT: $L__BB110_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM60-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM60-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM60-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM60-NEXT: setp.gt.u32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM60-NEXT: setp.gt.u32 %p5, %r12, %r11;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM60-NEXT: setp.gt.u32 %p6, %r14, %r13;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM60-NEXT: setp.gt.u32 %p7, %r16, %r15;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM60-NEXT: setp.gt.u32 %p8, %r18, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r8;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r9;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r10;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r11;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r12;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r13;
+; SM60-NEXT: cvt.u16.u32 %rs12, %r14;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r15;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r16;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r18;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM60-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM60-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM60-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM60-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM60-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM60-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM60-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM60-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM60-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB110_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: umax_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [umax_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB111_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: max.u16 %rs5, %rs4, %rs2;
+; SM60-NEXT: max.u16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB111_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: umax_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB112_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: max.u16 %rs5, %rs4, %rs2;
+; SM60-NEXT: max.u16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM60-NEXT: max.u16 %rs11, %rs10, %rs8;
+; SM60-NEXT: max.u16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB112_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: umax_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB113_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: max.u16 %rs5, %rs4, %rs2;
+; SM60-NEXT: max.u16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM60-NEXT: max.u16 %rs11, %rs10, %rs8;
+; SM60-NEXT: max.u16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB113_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: $L__BB113_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM60-NEXT: max.u16 %rs17, %rs16, %rs14;
+; SM60-NEXT: max.u16 %rs18, %rs15, %rs13;
+; SM60-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: max.u16 %rs23, %rs22, %rs20;
+; SM60-NEXT: max.u16 %rs24, %rs21, %rs19;
+; SM60-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB113_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @umax_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: umax_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.max.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @umax_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: umax_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.max.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.max.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.max.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @umax_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: umax_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umax_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.max.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.max.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.max.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.max.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.max.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.max.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.max.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @umax_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: umax_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.max.u64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @umax_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: umax_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.max.u64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.max.u64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.max.u64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @umax_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: umax_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umax_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umax_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.max.u64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.max.u64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.max.u64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.max.u64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.max.u64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.max.u64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.max.u64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: umin_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<18>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umin_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM60-NEXT: $L__BB120_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r17, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM60-NEXT: min.u16 %rs9, %rs5, %rs7;
+; SM60-NEXT: min.u16 %rs10, %rs6, %rs8;
+; SM60-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM60-NEXT: or.b16 %rs12, %rs9, %rs11;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM60-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NEXT: mov.b32 %r17, %r3;
+; SM60-NEXT: @%p1 bra $L__BB120_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: umin_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [umin_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM60-NEXT: $L__BB121_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
+; SM60-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
+; SM60-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
+; SM60-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
+; SM60-NEXT: setp.le.u32 %p4, %r10, %r9;
+; SM60-NEXT: selp.b32 %r11, %r10, %r9, %p4;
+; SM60-NEXT: selp.b32 %r12, %r8, %r7, %p3;
+; SM60-NEXT: prmt.b32 %r13, %r12, %r11, 0x3340U;
+; SM60-NEXT: selp.b32 %r14, %r6, %r5, %p2;
+; SM60-NEXT: selp.b32 %r15, %r4, %r3, %p1;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r14, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r13, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p5 bra $L__BB121_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: umin_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM60-NEXT: $L__BB122_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM60-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM60-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM60-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM60-NEXT: setp.le.u32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM60-NEXT: setp.le.u32 %p5, %r12, %r11;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM60-NEXT: setp.le.u32 %p6, %r14, %r13;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM60-NEXT: setp.le.u32 %p7, %r16, %r15;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM60-NEXT: setp.le.u32 %p8, %r18, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r8;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r9;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r10;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r11;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r12;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r13;
+; SM60-NEXT: cvt.u16.u32 %rs12, %r14;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r15;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r16;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r18;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM60-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM60-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM60-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM60-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM60-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM60-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM60-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM60-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM60-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB122_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: umin_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [umin_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB123_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: min.u16 %rs5, %rs4, %rs2;
+; SM60-NEXT: min.u16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB123_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: umin_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB124_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: min.u16 %rs5, %rs4, %rs2;
+; SM60-NEXT: min.u16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM60-NEXT: min.u16 %rs11, %rs10, %rs8;
+; SM60-NEXT: min.u16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB124_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: umin_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB125_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: min.u16 %rs5, %rs4, %rs2;
+; SM60-NEXT: min.u16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM60-NEXT: min.u16 %rs11, %rs10, %rs8;
+; SM60-NEXT: min.u16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB125_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: $L__BB125_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM60-NEXT: min.u16 %rs17, %rs16, %rs14;
+; SM60-NEXT: min.u16 %rs18, %rs15, %rs13;
+; SM60-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: min.u16 %rs23, %rs22, %rs20;
+; SM60-NEXT: min.u16 %rs24, %rs21, %rs19;
+; SM60-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB125_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @umin_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: umin_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.min.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @umin_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: umin_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.min.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.min.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.min.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @umin_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: umin_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umin_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.min.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.min.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.min.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.min.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.min.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.min.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.min.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @umin_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: umin_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.min.u64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @umin_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: umin_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.min.u64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.min.u64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.min.u64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @umin_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: umin_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umin_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umin_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.min.u64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.min.u64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.min.u64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.min.u64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.min.u64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.min.u64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.min.u64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<15>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [uinc_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs8, %rs9}, %r4;
+; SM60-NEXT: $L__BB132_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: and.b16 %rs4, %rs3, 255;
+; SM60-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM60-NEXT: add.s16 %rs6, %rs5, 1;
+; SM60-NEXT: add.s16 %rs7, %rs3, 1;
+; SM60-NEXT: setp.ge.u16 %p1, %rs5, %rs9;
+; SM60-NEXT: setp.ge.u16 %p2, %rs4, %rs8;
+; SM60-NEXT: selp.b16 %rs10, 0, %rs7, %p2;
+; SM60-NEXT: selp.b16 %rs11, 0, %rs6, %p1;
+; SM60-NEXT: shl.b16 %rs12, %rs11, 8;
+; SM60-NEXT: and.b16 %rs13, %rs10, 255;
+; SM60-NEXT: or.b16 %rs14, %rs13, %rs12;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs14;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p3 bra $L__BB132_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<23>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r22, [%rd1];
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
+; SM60-NEXT: $L__BB133_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r3, %r22, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: add.s16 %rs2, %rs1, 1;
+; SM60-NEXT: prmt.b32 %r4, %r22, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM60-NEXT: add.s16 %rs4, %rs3, 1;
+; SM60-NEXT: prmt.b32 %r5, %r22, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM60-NEXT: add.s16 %rs6, %rs5, 1;
+; SM60-NEXT: prmt.b32 %r6, %r22, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM60-NEXT: add.s16 %rs8, %rs7, 1;
+; SM60-NEXT: setp.ge.u32 %p1, %r3, %r7;
+; SM60-NEXT: setp.ge.u32 %p2, %r4, %r8;
+; SM60-NEXT: setp.ge.u32 %p3, %r5, %r9;
+; SM60-NEXT: setp.ge.u32 %p4, %r6, %r10;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs8;
+; SM60-NEXT: selp.b32 %r12, 0, %r11, %p4;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs6;
+; SM60-NEXT: selp.b32 %r14, 0, %r13, %p3;
+; SM60-NEXT: prmt.b32 %r15, %r14, %r12, 0x3340U;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs4;
+; SM60-NEXT: selp.b32 %r17, 0, %r16, %p2;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs2;
+; SM60-NEXT: selp.b32 %r19, 0, %r18, %p1;
+; SM60-NEXT: prmt.b32 %r20, %r19, %r17, 0x3340U;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r15, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r22;
+; SM60-NEXT: mov.b32 %r22, %r1;
+; SM60-NEXT: @%p5 bra $L__BB133_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM60-NEXT: $L__BB134_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r3, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: add.s16 %rs2, %rs1, 1;
+; SM60-NEXT: prmt.b32 %r4, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM60-NEXT: add.s16 %rs4, %rs3, 1;
+; SM60-NEXT: prmt.b32 %r5, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM60-NEXT: add.s16 %rs6, %rs5, 1;
+; SM60-NEXT: prmt.b32 %r6, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM60-NEXT: add.s16 %rs8, %rs7, 1;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r7;
+; SM60-NEXT: add.s16 %rs10, %rs9, 1;
+; SM60-NEXT: prmt.b32 %r8, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r8;
+; SM60-NEXT: add.s16 %rs12, %rs11, 1;
+; SM60-NEXT: prmt.b32 %r9, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r9;
+; SM60-NEXT: add.s16 %rs14, %rs13, 1;
+; SM60-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r10;
+; SM60-NEXT: add.s16 %rs16, %rs15, 1;
+; SM60-NEXT: setp.ge.u32 %p1, %r3, %r11;
+; SM60-NEXT: setp.ge.u32 %p2, %r4, %r12;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM60-NEXT: setp.ge.u32 %p3, %r5, %r13;
+; SM60-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM60-NEXT: setp.ge.u32 %p4, %r6, %r14;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0x7770U;
+; SM60-NEXT: setp.ge.u32 %p5, %r7, %r15;
+; SM60-NEXT: prmt.b32 %r16, %r1, 0, 0x7771U;
+; SM60-NEXT: setp.ge.u32 %p6, %r8, %r16;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0x7772U;
+; SM60-NEXT: setp.ge.u32 %p7, %r9, %r17;
+; SM60-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
+; SM60-NEXT: setp.ge.u32 %p8, %r10, %r18;
+; SM60-NEXT: selp.b16 %rs17, 0, %rs16, %p8;
+; SM60-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM60-NEXT: selp.b16 %rs18, 0, %rs14, %p7;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM60-NEXT: selp.b16 %rs19, 0, %rs12, %p6;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM60-NEXT: selp.b16 %rs20, 0, %rs10, %p5;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: selp.b16 %rs21, 0, %rs8, %p4;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM60-NEXT: selp.b16 %rs22, 0, %rs6, %p3;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM60-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM60-NEXT: selp.b16 %rs23, 0, %rs4, %p2;
+; SM60-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM60-NEXT: selp.b16 %rs24, 0, %rs2, %p1;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB134_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: $L__BB135_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs1, 1;
+; SM60-NEXT: add.s16 %rs4, %rs2, 1;
+; SM60-NEXT: setp.ge.u16 %p1, %rs1, %rs5;
+; SM60-NEXT: setp.ge.u16 %p2, %rs2, %rs6;
+; SM60-NEXT: selp.b16 %rs7, 0, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs8, 0, %rs3, %p1;
+; SM60-NEXT: mov.b32 %r3, {%rs8, %rs7};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p3 bra $L__BB135_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM60-NEXT: $L__BB136_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r6;
+; SM60-NEXT: add.s16 %rs3, %rs1, 1;
+; SM60-NEXT: add.s16 %rs4, %rs2, 1;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r5;
+; SM60-NEXT: add.s16 %rs7, %rs5, 1;
+; SM60-NEXT: add.s16 %rs8, %rs6, 1;
+; SM60-NEXT: setp.ge.u16 %p1, %rs1, %rs9;
+; SM60-NEXT: setp.ge.u16 %p2, %rs2, %rs10;
+; SM60-NEXT: setp.ge.u16 %p3, %rs5, %rs11;
+; SM60-NEXT: setp.ge.u16 %p4, %rs6, %rs12;
+; SM60-NEXT: selp.b16 %rs13, 0, %rs8, %p4;
+; SM60-NEXT: selp.b16 %rs14, 0, %rs7, %p3;
+; SM60-NEXT: mov.b32 %r3, {%rs14, %rs13};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: selp.b16 %rs15, 0, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs16, 0, %rs3, %p1;
+; SM60-NEXT: mov.b32 %r4, {%rs16, %rs15};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB136_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<33>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM60-NEXT: $L__BB137_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r10;
+; SM60-NEXT: add.s16 %rs3, %rs1, 1;
+; SM60-NEXT: add.s16 %rs4, %rs2, 1;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r9;
+; SM60-NEXT: add.s16 %rs7, %rs5, 1;
+; SM60-NEXT: add.s16 %rs8, %rs6, 1;
+; SM60-NEXT: setp.ge.u16 %p1, %rs1, %rs9;
+; SM60-NEXT: setp.ge.u16 %p2, %rs2, %rs10;
+; SM60-NEXT: setp.ge.u16 %p3, %rs5, %rs11;
+; SM60-NEXT: setp.ge.u16 %p4, %rs6, %rs12;
+; SM60-NEXT: selp.b16 %rs13, 0, %rs8, %p4;
+; SM60-NEXT: selp.b16 %rs14, 0, %rs7, %p3;
+; SM60-NEXT: mov.b32 %r5, {%rs14, %rs13};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: selp.b16 %rs15, 0, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs16, 0, %rs3, %p1;
+; SM60-NEXT: mov.b32 %r6, {%rs16, %rs15};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB137_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB137_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r12;
+; SM60-NEXT: add.s16 %rs19, %rs17, 1;
+; SM60-NEXT: add.s16 %rs20, %rs18, 1;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r11;
+; SM60-NEXT: add.s16 %rs23, %rs21, 1;
+; SM60-NEXT: add.s16 %rs24, %rs22, 1;
+; SM60-NEXT: mov.b32 {%rs25, %rs26}, %r4;
+; SM60-NEXT: setp.ge.u16 %p6, %rs17, %rs25;
+; SM60-NEXT: setp.ge.u16 %p7, %rs18, %rs26;
+; SM60-NEXT: mov.b32 {%rs27, %rs28}, %r3;
+; SM60-NEXT: setp.ge.u16 %p8, %rs21, %rs27;
+; SM60-NEXT: setp.ge.u16 %p9, %rs22, %rs28;
+; SM60-NEXT: selp.b16 %rs29, 0, %rs24, %p9;
+; SM60-NEXT: selp.b16 %rs30, 0, %rs23, %p8;
+; SM60-NEXT: mov.b32 %r7, {%rs30, %rs29};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: selp.b16 %rs31, 0, %rs20, %p7;
+; SM60-NEXT: selp.b16 %rs32, 0, %rs19, %p6;
+; SM60-NEXT: mov.b32 %r8, {%rs32, %rs31};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB137_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @uinc_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.inc.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.inc.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @uinc_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.inc.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.inc.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.inc.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.inc.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @uinc_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.inc.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.inc.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.inc.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.inc.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.inc.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.inc.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.inc.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.inc.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [uinc_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
+; SM60-NEXT: $L__BB141_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd6, %rd10, 1;
+; SM60-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
+; SM60-NEXT: selp.b64 %rd7, 0, %rd6, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM60-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
+; SM60-NEXT: mov.b64 %rd10, %rd3;
+; SM60-NEXT: @%p2 bra $L__BB141_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
+; SM60-NEXT: $L__BB141_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd8, %rd11, 1;
+; SM60-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
+; SM60-NEXT: selp.b64 %rd9, 0, %rd8, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM60-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
+; SM60-NEXT: mov.b64 %rd11, %rd4;
+; SM60-NEXT: @%p4 bra $L__BB141_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [uinc_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
+; SM60-NEXT: $L__BB142_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd18;
+; SM60-NEXT: add.s64 %rd10, %rd5, 1;
+; SM60-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
+; SM60-NEXT: selp.b64 %rd11, 0, %rd10, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
+; SM60-NEXT: @%p2 bra $L__BB142_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
+; SM60-NEXT: $L__BB142_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd19;
+; SM60-NEXT: add.s64 %rd12, %rd6, 1;
+; SM60-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
+; SM60-NEXT: selp.b64 %rd13, 0, %rd12, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
+; SM60-NEXT: @%p4 bra $L__BB142_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
+; SM60-NEXT: $L__BB142_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd14, %rd20, 1;
+; SM60-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
+; SM60-NEXT: selp.b64 %rd15, 0, %rd14, %p5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM60-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
+; SM60-NEXT: mov.b64 %rd20, %rd7;
+; SM60-NEXT: @%p6 bra $L__BB142_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
+; SM60-NEXT: $L__BB142_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd16, %rd21, 1;
+; SM60-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
+; SM60-NEXT: selp.b64 %rd17, 0, %rd16, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM60-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
+; SM60-NEXT: mov.b64 %rd21, %rd8;
+; SM60-NEXT: @%p8 bra $L__BB142_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<17>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [uinc_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
+; SM60-NEXT: $L__BB143_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd34;
+; SM60-NEXT: add.s64 %rd18, %rd7, 1;
+; SM60-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
+; SM60-NEXT: selp.b64 %rd19, 0, %rd18, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
+; SM60-NEXT: @%p2 bra $L__BB143_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
+; SM60-NEXT: $L__BB143_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd35;
+; SM60-NEXT: add.s64 %rd20, %rd8, 1;
+; SM60-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
+; SM60-NEXT: selp.b64 %rd21, 0, %rd20, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB143_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
+; SM60-NEXT: $L__BB143_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd36;
+; SM60-NEXT: add.s64 %rd22, %rd9, 1;
+; SM60-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
+; SM60-NEXT: selp.b64 %rd23, 0, %rd22, %p5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
+; SM60-NEXT: @%p6 bra $L__BB143_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
+; SM60-NEXT: $L__BB143_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd37;
+; SM60-NEXT: add.s64 %rd24, %rd10, 1;
+; SM60-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
+; SM60-NEXT: selp.b64 %rd25, 0, %rd24, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
+; SM60-NEXT: @%p8 bra $L__BB143_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
+; SM60-NEXT: $L__BB143_9: // %atomicrmw.start41
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd38;
+; SM60-NEXT: add.s64 %rd26, %rd13, 1;
+; SM60-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
+; SM60-NEXT: selp.b64 %rd27, 0, %rd26, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM60-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
+; SM60-NEXT: @%p10 bra $L__BB143_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end40
+; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
+; SM60-NEXT: $L__BB143_11: // %atomicrmw.start47
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd39;
+; SM60-NEXT: add.s64 %rd28, %rd14, 1;
+; SM60-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
+; SM60-NEXT: selp.b64 %rd29, 0, %rd28, %p11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM60-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
+; SM60-NEXT: @%p12 bra $L__BB143_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end46
+; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
+; SM60-NEXT: $L__BB143_13: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd30, %rd40, 1;
+; SM60-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
+; SM60-NEXT: selp.b64 %rd31, 0, %rd30, %p13;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM60-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
+; SM60-NEXT: mov.b64 %rd40, %rd15;
+; SM60-NEXT: @%p14 bra $L__BB143_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end57
+; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
+; SM60-NEXT: $L__BB143_15: // %atomicrmw.start64
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd32, %rd41, 1;
+; SM60-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
+; SM60-NEXT: selp.b64 %rd33, 0, %rd32, %p15;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM60-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
+; SM60-NEXT: mov.b64 %rd41, %rd16;
+; SM60-NEXT: @%p16 bra $L__BB143_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end63
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [udec_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs8, %rs9}, %r4;
+; SM60-NEXT: $L__BB144_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: and.b16 %rs4, %rs3, 255;
+; SM60-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM60-NEXT: add.s16 %rs6, %rs5, -1;
+; SM60-NEXT: add.s16 %rs7, %rs3, -1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs5, 0;
+; SM60-NEXT: setp.eq.b16 %p2, %rs4, 0;
+; SM60-NEXT: setp.gt.u16 %p3, %rs5, %rs9;
+; SM60-NEXT: setp.gt.u16 %p4, %rs4, %rs8;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs7, %p4;
+; SM60-NEXT: selp.b16 %rs11, %rs8, %rs10, %p2;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs13, %rs9, %rs12, %p1;
+; SM60-NEXT: shl.b16 %rs14, %rs13, 8;
+; SM60-NEXT: and.b16 %rs15, %rs11, 255;
+; SM60-NEXT: or.b16 %rs16, %rs15, %rs14;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs16;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p5, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p5 bra $L__BB144_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<27>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r26, [%rd1];
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
+; SM60-NEXT: $L__BB145_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r3, %r26, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: add.s16 %rs2, %rs1, -1;
+; SM60-NEXT: prmt.b32 %r4, %r26, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM60-NEXT: add.s16 %rs4, %rs3, -1;
+; SM60-NEXT: prmt.b32 %r5, %r26, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM60-NEXT: add.s16 %rs6, %rs5, -1;
+; SM60-NEXT: prmt.b32 %r6, %r26, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM60-NEXT: add.s16 %rs8, %rs7, -1;
+; SM60-NEXT: setp.eq.b32 %p1, %r3, 0;
+; SM60-NEXT: setp.eq.b32 %p2, %r4, 0;
+; SM60-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM60-NEXT: setp.eq.b32 %p4, %r6, 0;
+; SM60-NEXT: setp.gt.u32 %p5, %r3, %r7;
+; SM60-NEXT: setp.gt.u32 %p6, %r4, %r8;
+; SM60-NEXT: setp.gt.u32 %p7, %r5, %r9;
+; SM60-NEXT: setp.gt.u32 %p8, %r6, %r10;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs8;
+; SM60-NEXT: selp.b32 %r12, %r10, %r11, %p8;
+; SM60-NEXT: selp.b32 %r13, %r10, %r12, %p4;
+; SM60-NEXT: cvt.u32.u16 %r14, %rs6;
+; SM60-NEXT: selp.b32 %r15, %r9, %r14, %p7;
+; SM60-NEXT: selp.b32 %r16, %r9, %r15, %p3;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r13, 0x3340U;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs4;
+; SM60-NEXT: selp.b32 %r19, %r8, %r18, %p6;
+; SM60-NEXT: selp.b32 %r20, %r8, %r19, %p2;
+; SM60-NEXT: cvt.u32.u16 %r21, %rs2;
+; SM60-NEXT: selp.b32 %r22, %r7, %r21, %p5;
+; SM60-NEXT: selp.b32 %r23, %r7, %r22, %p1;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r17, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM60-NEXT: setp.ne.b32 %p9, %r1, %r26;
+; SM60-NEXT: mov.b32 %r26, %r1;
+; SM60-NEXT: @%p9 bra $L__BB145_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<18>;
+; SM60-NEXT: .reg .b16 %rs<41>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r11;
+; SM60-NEXT: cvt.u16.u32 %rs18, %r12;
+; SM60-NEXT: $L__BB146_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r3, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: add.s16 %rs2, %rs1, -1;
+; SM60-NEXT: prmt.b32 %r4, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM60-NEXT: add.s16 %rs4, %rs3, -1;
+; SM60-NEXT: prmt.b32 %r5, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM60-NEXT: add.s16 %rs6, %rs5, -1;
+; SM60-NEXT: prmt.b32 %r6, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM60-NEXT: add.s16 %rs8, %rs7, -1;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r7;
+; SM60-NEXT: add.s16 %rs10, %rs9, -1;
+; SM60-NEXT: prmt.b32 %r8, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r8;
+; SM60-NEXT: add.s16 %rs12, %rs11, -1;
+; SM60-NEXT: prmt.b32 %r9, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r9;
+; SM60-NEXT: add.s16 %rs14, %rs13, -1;
+; SM60-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r10;
+; SM60-NEXT: add.s16 %rs16, %rs15, -1;
+; SM60-NEXT: setp.eq.b32 %p1, %r3, 0;
+; SM60-NEXT: setp.eq.b32 %p2, %r4, 0;
+; SM60-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM60-NEXT: setp.eq.b32 %p4, %r6, 0;
+; SM60-NEXT: setp.eq.b32 %p5, %r7, 0;
+; SM60-NEXT: setp.eq.b32 %p6, %r8, 0;
+; SM60-NEXT: setp.eq.b32 %p7, %r9, 0;
+; SM60-NEXT: setp.eq.b32 %p8, %r10, 0;
+; SM60-NEXT: setp.gt.u32 %p9, %r3, %r11;
+; SM60-NEXT: setp.gt.u32 %p10, %r4, %r12;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM60-NEXT: setp.gt.u32 %p11, %r5, %r13;
+; SM60-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM60-NEXT: setp.gt.u32 %p12, %r6, %r14;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0x7770U;
+; SM60-NEXT: setp.gt.u32 %p13, %r7, %r15;
+; SM60-NEXT: prmt.b32 %r16, %r1, 0, 0x7771U;
+; SM60-NEXT: setp.gt.u32 %p14, %r8, %r16;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0x7772U;
+; SM60-NEXT: setp.gt.u32 %p15, %r9, %r17;
+; SM60-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
+; SM60-NEXT: setp.gt.u32 %p16, %r10, %r18;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r13;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r14;
+; SM60-NEXT: cvt.u16.u32 %rs21, %r15;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r16;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs24, %r18;
+; SM60-NEXT: selp.b16 %rs25, %rs24, %rs16, %p16;
+; SM60-NEXT: selp.b16 %rs26, %rs24, %rs25, %p8;
+; SM60-NEXT: cvt.u32.u16 %r19, %rs26;
+; SM60-NEXT: selp.b16 %rs27, %rs23, %rs14, %p15;
+; SM60-NEXT: selp.b16 %rs28, %rs23, %rs27, %p7;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs28;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM60-NEXT: selp.b16 %rs29, %rs22, %rs12, %p14;
+; SM60-NEXT: selp.b16 %rs30, %rs22, %rs29, %p6;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs30;
+; SM60-NEXT: selp.b16 %rs31, %rs21, %rs10, %p13;
+; SM60-NEXT: selp.b16 %rs32, %rs21, %rs31, %p5;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs32;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: selp.b16 %rs33, %rs20, %rs8, %p12;
+; SM60-NEXT: selp.b16 %rs34, %rs20, %rs33, %p4;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs34;
+; SM60-NEXT: selp.b16 %rs35, %rs19, %rs6, %p11;
+; SM60-NEXT: selp.b16 %rs36, %rs19, %rs35, %p3;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs36;
+; SM60-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM60-NEXT: selp.b16 %rs37, %rs18, %rs4, %p10;
+; SM60-NEXT: selp.b16 %rs38, %rs18, %rs37, %p2;
+; SM60-NEXT: cvt.u32.u16 %r29, %rs38;
+; SM60-NEXT: selp.b16 %rs39, %rs17, %rs2, %p9;
+; SM60-NEXT: selp.b16 %rs40, %rs17, %rs39, %p1;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs40;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p17, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p17 bra $L__BB146_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<11>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: $L__BB147_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs1, -1;
+; SM60-NEXT: add.s16 %rs4, %rs2, -1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM60-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM60-NEXT: setp.gt.u16 %p3, %rs1, %rs5;
+; SM60-NEXT: setp.gt.u16 %p4, %rs2, %rs6;
+; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p4;
+; SM60-NEXT: selp.b16 %rs8, %rs6, %rs7, %p2;
+; SM60-NEXT: selp.b16 %rs9, %rs5, %rs3, %p3;
+; SM60-NEXT: selp.b16 %rs10, %rs5, %rs9, %p1;
+; SM60-NEXT: mov.b32 %r3, {%rs10, %rs8};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p5 bra $L__BB147_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<21>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM60-NEXT: $L__BB148_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r6;
+; SM60-NEXT: add.s16 %rs3, %rs1, -1;
+; SM60-NEXT: add.s16 %rs4, %rs2, -1;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r5;
+; SM60-NEXT: add.s16 %rs7, %rs5, -1;
+; SM60-NEXT: add.s16 %rs8, %rs6, -1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM60-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM60-NEXT: setp.eq.b16 %p3, %rs5, 0;
+; SM60-NEXT: setp.eq.b16 %p4, %rs6, 0;
+; SM60-NEXT: setp.gt.u16 %p5, %rs1, %rs9;
+; SM60-NEXT: setp.gt.u16 %p6, %rs2, %rs10;
+; SM60-NEXT: setp.gt.u16 %p7, %rs5, %rs11;
+; SM60-NEXT: setp.gt.u16 %p8, %rs6, %rs12;
+; SM60-NEXT: selp.b16 %rs13, %rs12, %rs8, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs12, %rs13, %p4;
+; SM60-NEXT: selp.b16 %rs15, %rs11, %rs7, %p7;
+; SM60-NEXT: selp.b16 %rs16, %rs11, %rs15, %p3;
+; SM60-NEXT: mov.b32 %r3, {%rs16, %rs14};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: selp.b16 %rs17, %rs10, %rs4, %p6;
+; SM60-NEXT: selp.b16 %rs18, %rs10, %rs17, %p2;
+; SM60-NEXT: selp.b16 %rs19, %rs9, %rs3, %p5;
+; SM60-NEXT: selp.b16 %rs20, %rs9, %rs19, %p1;
+; SM60-NEXT: mov.b32 %r4, {%rs20, %rs18};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB148_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<19>;
+; SM60-NEXT: .reg .b16 %rs<41>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM60-NEXT: $L__BB149_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r10;
+; SM60-NEXT: add.s16 %rs3, %rs1, -1;
+; SM60-NEXT: add.s16 %rs4, %rs2, -1;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r9;
+; SM60-NEXT: add.s16 %rs7, %rs5, -1;
+; SM60-NEXT: add.s16 %rs8, %rs6, -1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM60-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM60-NEXT: setp.eq.b16 %p3, %rs5, 0;
+; SM60-NEXT: setp.eq.b16 %p4, %rs6, 0;
+; SM60-NEXT: setp.gt.u16 %p5, %rs1, %rs9;
+; SM60-NEXT: setp.gt.u16 %p6, %rs2, %rs10;
+; SM60-NEXT: setp.gt.u16 %p7, %rs5, %rs11;
+; SM60-NEXT: setp.gt.u16 %p8, %rs6, %rs12;
+; SM60-NEXT: selp.b16 %rs13, %rs12, %rs8, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs12, %rs13, %p4;
+; SM60-NEXT: selp.b16 %rs15, %rs11, %rs7, %p7;
+; SM60-NEXT: selp.b16 %rs16, %rs11, %rs15, %p3;
+; SM60-NEXT: mov.b32 %r5, {%rs16, %rs14};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: selp.b16 %rs17, %rs10, %rs4, %p6;
+; SM60-NEXT: selp.b16 %rs18, %rs10, %rs17, %p2;
+; SM60-NEXT: selp.b16 %rs19, %rs9, %rs3, %p5;
+; SM60-NEXT: selp.b16 %rs20, %rs9, %rs19, %p1;
+; SM60-NEXT: mov.b32 %r6, {%rs20, %rs18};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB149_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB149_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: add.s16 %rs23, %rs21, -1;
+; SM60-NEXT: add.s16 %rs24, %rs22, -1;
+; SM60-NEXT: mov.b32 {%rs25, %rs26}, %r11;
+; SM60-NEXT: add.s16 %rs27, %rs25, -1;
+; SM60-NEXT: add.s16 %rs28, %rs26, -1;
+; SM60-NEXT: setp.eq.b16 %p10, %rs21, 0;
+; SM60-NEXT: setp.eq.b16 %p11, %rs22, 0;
+; SM60-NEXT: setp.eq.b16 %p12, %rs25, 0;
+; SM60-NEXT: setp.eq.b16 %p13, %rs26, 0;
+; SM60-NEXT: mov.b32 {%rs29, %rs30}, %r4;
+; SM60-NEXT: setp.gt.u16 %p14, %rs21, %rs29;
+; SM60-NEXT: setp.gt.u16 %p15, %rs22, %rs30;
+; SM60-NEXT: mov.b32 {%rs31, %rs32}, %r3;
+; SM60-NEXT: setp.gt.u16 %p16, %rs25, %rs31;
+; SM60-NEXT: setp.gt.u16 %p17, %rs26, %rs32;
+; SM60-NEXT: selp.b16 %rs33, %rs32, %rs28, %p17;
+; SM60-NEXT: selp.b16 %rs34, %rs32, %rs33, %p13;
+; SM60-NEXT: selp.b16 %rs35, %rs31, %rs27, %p16;
+; SM60-NEXT: selp.b16 %rs36, %rs31, %rs35, %p12;
+; SM60-NEXT: mov.b32 %r7, {%rs36, %rs34};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: selp.b16 %rs37, %rs30, %rs24, %p15;
+; SM60-NEXT: selp.b16 %rs38, %rs30, %rs37, %p11;
+; SM60-NEXT: selp.b16 %rs39, %rs29, %rs23, %p14;
+; SM60-NEXT: selp.b16 %rs40, %rs29, %rs39, %p10;
+; SM60-NEXT: mov.b32 %r8, {%rs40, %rs38};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p18 bra $L__BB149_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @udec_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.dec.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.dec.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @udec_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.dec.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.dec.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.dec.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.dec.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @udec_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [udec_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.dec.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.dec.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.dec.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.dec.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.dec.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.dec.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.dec.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.dec.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<7>;
+; SM60-NEXT: .reg .b64 %rd<14>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [udec_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd5];
+; SM60-NEXT: $L__BB153_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd6, %rd12, -1;
+; SM60-NEXT: setp.eq.b64 %p1, %rd12, 0;
+; SM60-NEXT: setp.gt.u64 %p2, %rd12, %rd1;
+; SM60-NEXT: selp.b64 %rd7, %rd1, %rd6, %p2;
+; SM60-NEXT: selp.b64 %rd8, %rd1, %rd7, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd12, %rd8;
+; SM60-NEXT: setp.ne.b64 %p3, %rd3, %rd12;
+; SM60-NEXT: mov.b64 %rd12, %rd3;
+; SM60-NEXT: @%p3 bra $L__BB153_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd5+8];
+; SM60-NEXT: $L__BB153_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd9, %rd13, -1;
+; SM60-NEXT: setp.eq.b64 %p4, %rd13, 0;
+; SM60-NEXT: setp.gt.u64 %p5, %rd13, %rd2;
+; SM60-NEXT: selp.b64 %rd10, %rd2, %rd9, %p5;
+; SM60-NEXT: selp.b64 %rd11, %rd2, %rd10, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd13, %rd11;
+; SM60-NEXT: setp.ne.b64 %p6, %rd4, %rd13;
+; SM60-NEXT: mov.b64 %rd13, %rd4;
+; SM60-NEXT: @%p6 bra $L__BB153_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<13>;
+; SM60-NEXT: .reg .b64 %rd<26>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [udec_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd9];
+; SM60-NEXT: $L__BB154_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd22;
+; SM60-NEXT: add.s64 %rd10, %rd5, -1;
+; SM60-NEXT: setp.eq.b64 %p1, %rd5, 0;
+; SM60-NEXT: setp.gt.u64 %p2, %rd5, %rd3;
+; SM60-NEXT: selp.b64 %rd11, %rd3, %rd10, %p2;
+; SM60-NEXT: selp.b64 %rd12, %rd3, %rd11, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd22, [%rd9], %rd5, %rd12;
+; SM60-NEXT: setp.ne.b64 %p3, %rd22, %rd5;
+; SM60-NEXT: @%p3 bra $L__BB154_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd23, [%rd9+8];
+; SM60-NEXT: $L__BB154_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd23;
+; SM60-NEXT: add.s64 %rd13, %rd6, -1;
+; SM60-NEXT: setp.eq.b64 %p4, %rd6, 0;
+; SM60-NEXT: setp.gt.u64 %p5, %rd6, %rd4;
+; SM60-NEXT: selp.b64 %rd14, %rd4, %rd13, %p5;
+; SM60-NEXT: selp.b64 %rd15, %rd4, %rd14, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd9+8], %rd6, %rd15;
+; SM60-NEXT: setp.ne.b64 %p6, %rd23, %rd6;
+; SM60-NEXT: @%p6 bra $L__BB154_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd9+16];
+; SM60-NEXT: $L__BB154_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd16, %rd24, -1;
+; SM60-NEXT: setp.eq.b64 %p7, %rd24, 0;
+; SM60-NEXT: setp.gt.u64 %p8, %rd24, %rd1;
+; SM60-NEXT: selp.b64 %rd17, %rd1, %rd16, %p8;
+; SM60-NEXT: selp.b64 %rd18, %rd1, %rd17, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd24, %rd18;
+; SM60-NEXT: setp.ne.b64 %p9, %rd7, %rd24;
+; SM60-NEXT: mov.b64 %rd24, %rd7;
+; SM60-NEXT: @%p9 bra $L__BB154_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd25, [%rd9+24];
+; SM60-NEXT: $L__BB154_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd19, %rd25, -1;
+; SM60-NEXT: setp.eq.b64 %p10, %rd25, 0;
+; SM60-NEXT: setp.gt.u64 %p11, %rd25, %rd2;
+; SM60-NEXT: selp.b64 %rd20, %rd2, %rd19, %p11;
+; SM60-NEXT: selp.b64 %rd21, %rd2, %rd20, %p10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd25, %rd21;
+; SM60-NEXT: setp.ne.b64 %p12, %rd8, %rd25;
+; SM60-NEXT: mov.b64 %rd25, %rd8;
+; SM60-NEXT: @%p12 bra $L__BB154_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd22, %rd23};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<25>;
+; SM60-NEXT: .reg .b64 %rd<50>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [udec_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd42, [%rd17];
+; SM60-NEXT: $L__BB155_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd42;
+; SM60-NEXT: add.s64 %rd18, %rd7, -1;
+; SM60-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM60-NEXT: setp.gt.u64 %p2, %rd7, %rd5;
+; SM60-NEXT: selp.b64 %rd19, %rd5, %rd18, %p2;
+; SM60-NEXT: selp.b64 %rd20, %rd5, %rd19, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd42, [%rd17], %rd7, %rd20;
+; SM60-NEXT: setp.ne.b64 %p3, %rd42, %rd7;
+; SM60-NEXT: @%p3 bra $L__BB155_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd43, [%rd17+8];
+; SM60-NEXT: $L__BB155_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd43;
+; SM60-NEXT: add.s64 %rd21, %rd8, -1;
+; SM60-NEXT: setp.eq.b64 %p4, %rd8, 0;
+; SM60-NEXT: setp.gt.u64 %p5, %rd8, %rd6;
+; SM60-NEXT: selp.b64 %rd22, %rd6, %rd21, %p5;
+; SM60-NEXT: selp.b64 %rd23, %rd6, %rd22, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd43, [%rd17+8], %rd8, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd43, %rd8;
+; SM60-NEXT: @%p6 bra $L__BB155_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd44, [%rd17+16];
+; SM60-NEXT: $L__BB155_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd44;
+; SM60-NEXT: add.s64 %rd24, %rd9, -1;
+; SM60-NEXT: setp.eq.b64 %p7, %rd9, 0;
+; SM60-NEXT: setp.gt.u64 %p8, %rd9, %rd3;
+; SM60-NEXT: selp.b64 %rd25, %rd3, %rd24, %p8;
+; SM60-NEXT: selp.b64 %rd26, %rd3, %rd25, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd44, [%rd17+16], %rd9, %rd26;
+; SM60-NEXT: setp.ne.b64 %p9, %rd44, %rd9;
+; SM60-NEXT: @%p9 bra $L__BB155_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: ld.volatile.global.b64 %rd45, [%rd17+24];
+; SM60-NEXT: $L__BB155_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd45;
+; SM60-NEXT: add.s64 %rd27, %rd10, -1;
+; SM60-NEXT: setp.eq.b64 %p10, %rd10, 0;
+; SM60-NEXT: setp.gt.u64 %p11, %rd10, %rd4;
+; SM60-NEXT: selp.b64 %rd28, %rd4, %rd27, %p11;
+; SM60-NEXT: selp.b64 %rd29, %rd4, %rd28, %p10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd17+24], %rd10, %rd29;
+; SM60-NEXT: setp.ne.b64 %p12, %rd45, %rd10;
+; SM60-NEXT: @%p12 bra $L__BB155_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: ld.volatile.global.b64 %rd46, [%rd17+32];
+; SM60-NEXT: $L__BB155_9: // %atomicrmw.start41
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd46;
+; SM60-NEXT: add.s64 %rd30, %rd13, -1;
+; SM60-NEXT: setp.eq.b64 %p13, %rd13, 0;
+; SM60-NEXT: setp.gt.u64 %p14, %rd13, %rd1;
+; SM60-NEXT: selp.b64 %rd31, %rd1, %rd30, %p14;
+; SM60-NEXT: selp.b64 %rd32, %rd1, %rd31, %p13;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd46, [%rd17+32], %rd13, %rd32;
+; SM60-NEXT: setp.ne.b64 %p15, %rd46, %rd13;
+; SM60-NEXT: @%p15 bra $L__BB155_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end40
+; SM60-NEXT: ld.volatile.global.b64 %rd47, [%rd17+40];
+; SM60-NEXT: $L__BB155_11: // %atomicrmw.start47
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd47;
+; SM60-NEXT: add.s64 %rd33, %rd14, -1;
+; SM60-NEXT: setp.eq.b64 %p16, %rd14, 0;
+; SM60-NEXT: setp.gt.u64 %p17, %rd14, %rd2;
+; SM60-NEXT: selp.b64 %rd34, %rd2, %rd33, %p17;
+; SM60-NEXT: selp.b64 %rd35, %rd2, %rd34, %p16;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd47, [%rd17+40], %rd14, %rd35;
+; SM60-NEXT: setp.ne.b64 %p18, %rd47, %rd14;
+; SM60-NEXT: @%p18 bra $L__BB155_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end46
+; SM60-NEXT: ld.volatile.global.b64 %rd48, [%rd17+48];
+; SM60-NEXT: $L__BB155_13: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd36, %rd48, -1;
+; SM60-NEXT: setp.eq.b64 %p19, %rd48, 0;
+; SM60-NEXT: setp.gt.u64 %p20, %rd48, %rd11;
+; SM60-NEXT: selp.b64 %rd37, %rd11, %rd36, %p20;
+; SM60-NEXT: selp.b64 %rd38, %rd11, %rd37, %p19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd48, %rd38;
+; SM60-NEXT: setp.ne.b64 %p21, %rd15, %rd48;
+; SM60-NEXT: mov.b64 %rd48, %rd15;
+; SM60-NEXT: @%p21 bra $L__BB155_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end57
+; SM60-NEXT: ld.volatile.global.b64 %rd49, [%rd17+56];
+; SM60-NEXT: $L__BB155_15: // %atomicrmw.start64
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd39, %rd49, -1;
+; SM60-NEXT: setp.eq.b64 %p22, %rd49, 0;
+; SM60-NEXT: setp.gt.u64 %p23, %rd49, %rd12;
+; SM60-NEXT: selp.b64 %rd40, %rd12, %rd39, %p23;
+; SM60-NEXT: selp.b64 %rd41, %rd12, %rd40, %p22;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd49, %rd41;
+; SM60-NEXT: setp.ne.b64 %p24, %rd16, %rd49;
+; SM60-NEXT: mov.b64 %rd49, %rd16;
+; SM60-NEXT: @%p24 bra $L__BB155_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end63
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd46, %rd47};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<15>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_cond_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs6, %rs7}, %r4;
+; SM60-NEXT: $L__BB156_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: and.b16 %rs4, %rs3, 255;
+; SM60-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM60-NEXT: setp.ge.u16 %p1, %rs5, %rs7;
+; SM60-NEXT: setp.ge.u16 %p2, %rs4, %rs6;
+; SM60-NEXT: sub.s16 %rs8, %rs5, %rs7;
+; SM60-NEXT: sub.s16 %rs9, %rs3, %rs6;
+; SM60-NEXT: selp.b16 %rs10, %rs9, %rs3, %p2;
+; SM60-NEXT: selp.b16 %rs11, %rs8, %rs5, %p1;
+; SM60-NEXT: shl.b16 %rs12, %rs11, 8;
+; SM60-NEXT: and.b16 %rs13, %rs10, 255;
+; SM60-NEXT: or.b16 %rs14, %rs13, %rs12;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs14;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p3 bra $L__BB156_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<23>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r22, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r9;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r3;
+; SM60-NEXT: $L__BB157_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r22, 0, 0x7770U;
+; SM60-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r22, 0, 0x7771U;
+; SM60-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r8, %r22, 0, 0x7772U;
+; SM60-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r10, %r22, 0, 0x7773U;
+; SM60-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r4;
+; SM60-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM60-NEXT: sub.s16 %rs8, %rs7, %rs3;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r8;
+; SM60-NEXT: sub.s16 %rs10, %rs9, %rs2;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r10;
+; SM60-NEXT: sub.s16 %rs12, %rs11, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs12;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p4;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs10;
+; SM60-NEXT: selp.b32 %r14, %r13, %r8, %p3;
+; SM60-NEXT: prmt.b32 %r15, %r14, %r12, 0x3340U;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs8;
+; SM60-NEXT: selp.b32 %r17, %r16, %r6, %p2;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs6;
+; SM60-NEXT: selp.b32 %r19, %r18, %r4, %p1;
+; SM60-NEXT: prmt.b32 %r20, %r19, %r17, 0x3340U;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r15, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r22;
+; SM60-NEXT: mov.b32 %r22, %r1;
+; SM60-NEXT: @%p5 bra $L__BB157_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<33>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r5;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r3;
+; SM60-NEXT: $L__BB158_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM60-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM60-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM60-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM60-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM60-NEXT: setp.ge.u32 %p5, %r12, %r11;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM60-NEXT: setp.ge.u32 %p6, %r14, %r13;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM60-NEXT: setp.ge.u32 %p7, %r16, %r15;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM60-NEXT: setp.ge.u32 %p8, %r18, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r18;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r15;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r16;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r13;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r14;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r11;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r12;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r9;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r10;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r7;
+; SM60-NEXT: cvt.u16.u32 %rs12, %r8;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r6;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r4;
+; SM60-NEXT: sub.s16 %rs17, %rs16, %rs15;
+; SM60-NEXT: sub.s16 %rs18, %rs14, %rs13;
+; SM60-NEXT: sub.s16 %rs19, %rs12, %rs11;
+; SM60-NEXT: sub.s16 %rs20, %rs10, %rs9;
+; SM60-NEXT: sub.s16 %rs21, %rs8, %rs7;
+; SM60-NEXT: sub.s16 %rs22, %rs6, %rs5;
+; SM60-NEXT: sub.s16 %rs23, %rs4, %rs3;
+; SM60-NEXT: sub.s16 %rs24, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs25, %rs24, %rs2, %p8;
+; SM60-NEXT: cvt.u32.u16 %r19, %rs25;
+; SM60-NEXT: selp.b16 %rs26, %rs23, %rs4, %p7;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs26;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM60-NEXT: selp.b16 %rs27, %rs22, %rs6, %p6;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs27;
+; SM60-NEXT: selp.b16 %rs28, %rs21, %rs8, %p5;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs28;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: selp.b16 %rs29, %rs20, %rs10, %p4;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs29;
+; SM60-NEXT: selp.b16 %rs30, %rs19, %rs12, %p3;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs30;
+; SM60-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM60-NEXT: selp.b16 %rs31, %rs18, %rs14, %p2;
+; SM60-NEXT: cvt.u32.u16 %r29, %rs31;
+; SM60-NEXT: selp.b16 %rs32, %rs17, %rs16, %p1;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs32;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB158_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB159_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM60-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs5, %rs3, %rs1;
+; SM60-NEXT: sub.s16 %rs6, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs3, %p1;
+; SM60-NEXT: mov.b32 %r3, {%rs8, %rs7};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p3 bra $L__BB159_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; SM60-NEXT: $L__BB160_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r6;
+; SM60-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM60-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r5;
+; SM60-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM60-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM60-NEXT: sub.s16 %rs9, %rs3, %rs1;
+; SM60-NEXT: sub.s16 %rs10, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs11, %rs7, %rs5;
+; SM60-NEXT: sub.s16 %rs12, %rs8, %rs6;
+; SM60-NEXT: selp.b16 %rs13, %rs12, %rs8, %p4;
+; SM60-NEXT: selp.b16 %rs14, %rs11, %rs7, %p3;
+; SM60-NEXT: mov.b32 %r3, {%rs14, %rs13};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: selp.b16 %rs15, %rs10, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs16, %rs9, %rs3, %p1;
+; SM60-NEXT: mov.b32 %r4, {%rs16, %rs15};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB160_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<33>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; SM60-NEXT: $L__BB161_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r10;
+; SM60-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM60-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r9;
+; SM60-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM60-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM60-NEXT: sub.s16 %rs9, %rs3, %rs1;
+; SM60-NEXT: sub.s16 %rs10, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs11, %rs7, %rs5;
+; SM60-NEXT: sub.s16 %rs12, %rs8, %rs6;
+; SM60-NEXT: selp.b16 %rs13, %rs12, %rs8, %p4;
+; SM60-NEXT: selp.b16 %rs14, %rs11, %rs7, %p3;
+; SM60-NEXT: mov.b32 %r5, {%rs14, %rs13};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: selp.b16 %rs15, %rs10, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs16, %rs9, %rs3, %p1;
+; SM60-NEXT: mov.b32 %r6, {%rs16, %rs15};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB161_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB161_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r4;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r12;
+; SM60-NEXT: setp.ge.u16 %p6, %rs19, %rs17;
+; SM60-NEXT: setp.ge.u16 %p7, %rs20, %rs18;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r3;
+; SM60-NEXT: mov.b32 {%rs23, %rs24}, %r11;
+; SM60-NEXT: setp.ge.u16 %p8, %rs23, %rs21;
+; SM60-NEXT: setp.ge.u16 %p9, %rs24, %rs22;
+; SM60-NEXT: sub.s16 %rs25, %rs19, %rs17;
+; SM60-NEXT: sub.s16 %rs26, %rs20, %rs18;
+; SM60-NEXT: sub.s16 %rs27, %rs23, %rs21;
+; SM60-NEXT: sub.s16 %rs28, %rs24, %rs22;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs24, %p9;
+; SM60-NEXT: selp.b16 %rs30, %rs27, %rs23, %p8;
+; SM60-NEXT: mov.b32 %r7, {%rs30, %rs29};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: selp.b16 %rs31, %rs26, %rs20, %p7;
+; SM60-NEXT: selp.b16 %rs32, %rs25, %rs19, %p6;
+; SM60-NEXT: mov.b32 %r8, {%rs32, %rs31};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB161_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB162_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: setp.ge.u32 %p1, %r8, %r2;
+; SM60-NEXT: setp.ge.u32 %p2, %r7, %r1;
+; SM60-NEXT: sub.s32 %r3, %r8, %r2;
+; SM60-NEXT: sub.s32 %r4, %r7, %r1;
+; SM60-NEXT: selp.b32 %r5, %r4, %r7, %p2;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: selp.b32 %r6, %r3, %r8, %p1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p3 bra $L__BB162_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<7>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM60-NEXT: $L__BB163_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: setp.ge.u32 %p1, %r14, %r2;
+; SM60-NEXT: setp.ge.u32 %p2, %r13, %r1;
+; SM60-NEXT: sub.s32 %r5, %r14, %r2;
+; SM60-NEXT: sub.s32 %r6, %r13, %r1;
+; SM60-NEXT: selp.b32 %r7, %r6, %r13, %p2;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM60-NEXT: selp.b32 %r8, %r5, %r14, %p1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM60-NEXT: @%p3 bra $L__BB163_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM60-NEXT: $L__BB163_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r15;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r16;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: setp.ge.u32 %p4, %r16, %r4;
+; SM60-NEXT: setp.ge.u32 %p5, %r15, %r3;
+; SM60-NEXT: sub.s32 %r9, %r16, %r4;
+; SM60-NEXT: sub.s32 %r10, %r15, %r3;
+; SM60-NEXT: selp.b32 %r11, %r10, %r15, %p5;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: selp.b32 %r12, %r9, %r16, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p6, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd21;
+; SM60-NEXT: @%p6 bra $L__BB163_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<13>;
+; SM60-NEXT: .reg .b32 %r<33>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_cond_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM60-NEXT: $L__BB164_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r26;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: setp.ge.u32 %p1, %r26, %r6;
+; SM60-NEXT: setp.ge.u32 %p2, %r25, %r5;
+; SM60-NEXT: sub.s32 %r9, %r26, %r6;
+; SM60-NEXT: sub.s32 %r10, %r25, %r5;
+; SM60-NEXT: selp.b32 %r11, %r10, %r25, %p2;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r11;
+; SM60-NEXT: selp.b32 %r12, %r9, %r26, %p1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r12;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM60-NEXT: @%p3 bra $L__BB164_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM60-NEXT: $L__BB164_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r27;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: setp.ge.u32 %p4, %r28, %r8;
+; SM60-NEXT: setp.ge.u32 %p5, %r27, %r7;
+; SM60-NEXT: sub.s32 %r13, %r28, %r8;
+; SM60-NEXT: sub.s32 %r14, %r27, %r7;
+; SM60-NEXT: selp.b32 %r15, %r14, %r27, %p5;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM60-NEXT: selp.b32 %r16, %r13, %r28, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p6, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM60-NEXT: @%p6 bra $L__BB164_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd22;
+; SM60-NEXT: $L__BB164_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd23, %r29;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r30;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: setp.ge.u32 %p7, %r30, %r2;
+; SM60-NEXT: setp.ge.u32 %p8, %r29, %r1;
+; SM60-NEXT: sub.s32 %r17, %r30, %r2;
+; SM60-NEXT: sub.s32 %r18, %r29, %r1;
+; SM60-NEXT: selp.b32 %r19, %r18, %r29, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r19;
+; SM60-NEXT: selp.b32 %r20, %r17, %r30, %p7;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r20;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd26, %rd30;
+; SM60-NEXT: setp.ne.b64 %p9, %rd31, %rd26;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd31;
+; SM60-NEXT: @%p9 bra $L__BB164_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r31, %rd32;
+; SM60-NEXT: $L__BB164_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd33, %r31;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r32;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: setp.ge.u32 %p10, %r32, %r4;
+; SM60-NEXT: setp.ge.u32 %p11, %r31, %r3;
+; SM60-NEXT: sub.s32 %r21, %r32, %r4;
+; SM60-NEXT: sub.s32 %r22, %r31, %r3;
+; SM60-NEXT: selp.b32 %r23, %r22, %r31, %p11;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM60-NEXT: selp.b32 %r24, %r21, %r32, %p10;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd36, %rd40;
+; SM60-NEXT: setp.ne.b64 %p12, %rd41, %rd36;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r31, %rd41;
+; SM60-NEXT: @%p12 bra $L__BB164_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [usub_cond_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
+; SM60-NEXT: $L__BB165_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
+; SM60-NEXT: sub.s64 %rd6, %rd10, %rd1;
+; SM60-NEXT: selp.b64 %rd7, %rd6, %rd10, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM60-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
+; SM60-NEXT: mov.b64 %rd10, %rd3;
+; SM60-NEXT: @%p2 bra $L__BB165_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
+; SM60-NEXT: $L__BB165_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
+; SM60-NEXT: sub.s64 %rd8, %rd11, %rd2;
+; SM60-NEXT: selp.b64 %rd9, %rd8, %rd11, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM60-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
+; SM60-NEXT: mov.b64 %rd11, %rd4;
+; SM60-NEXT: @%p4 bra $L__BB165_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [usub_cond_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
+; SM60-NEXT: $L__BB166_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd18;
+; SM60-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
+; SM60-NEXT: sub.s64 %rd10, %rd5, %rd3;
+; SM60-NEXT: selp.b64 %rd11, %rd10, %rd5, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
+; SM60-NEXT: @%p2 bra $L__BB166_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
+; SM60-NEXT: $L__BB166_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd19;
+; SM60-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
+; SM60-NEXT: sub.s64 %rd12, %rd6, %rd4;
+; SM60-NEXT: selp.b64 %rd13, %rd12, %rd6, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
+; SM60-NEXT: @%p4 bra $L__BB166_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
+; SM60-NEXT: $L__BB166_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
+; SM60-NEXT: sub.s64 %rd14, %rd20, %rd1;
+; SM60-NEXT: selp.b64 %rd15, %rd14, %rd20, %p5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM60-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
+; SM60-NEXT: mov.b64 %rd20, %rd7;
+; SM60-NEXT: @%p6 bra $L__BB166_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
+; SM60-NEXT: $L__BB166_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
+; SM60-NEXT: sub.s64 %rd16, %rd21, %rd2;
+; SM60-NEXT: selp.b64 %rd17, %rd16, %rd21, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM60-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
+; SM60-NEXT: mov.b64 %rd21, %rd8;
+; SM60-NEXT: @%p8 bra $L__BB166_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<17>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [usub_cond_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [usub_cond_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
+; SM60-NEXT: $L__BB167_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd34;
+; SM60-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
+; SM60-NEXT: sub.s64 %rd18, %rd7, %rd5;
+; SM60-NEXT: selp.b64 %rd19, %rd18, %rd7, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
+; SM60-NEXT: @%p2 bra $L__BB167_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
+; SM60-NEXT: $L__BB167_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd35;
+; SM60-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
+; SM60-NEXT: sub.s64 %rd20, %rd8, %rd6;
+; SM60-NEXT: selp.b64 %rd21, %rd20, %rd8, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB167_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
+; SM60-NEXT: $L__BB167_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd36;
+; SM60-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
+; SM60-NEXT: sub.s64 %rd22, %rd9, %rd3;
+; SM60-NEXT: selp.b64 %rd23, %rd22, %rd9, %p5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
+; SM60-NEXT: @%p6 bra $L__BB167_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
+; SM60-NEXT: $L__BB167_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd37;
+; SM60-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
+; SM60-NEXT: sub.s64 %rd24, %rd10, %rd4;
+; SM60-NEXT: selp.b64 %rd25, %rd24, %rd10, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
+; SM60-NEXT: @%p8 bra $L__BB167_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
+; SM60-NEXT: $L__BB167_9: // %atomicrmw.start41
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd38;
+; SM60-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
+; SM60-NEXT: sub.s64 %rd26, %rd13, %rd1;
+; SM60-NEXT: selp.b64 %rd27, %rd26, %rd13, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM60-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
+; SM60-NEXT: @%p10 bra $L__BB167_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end40
+; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
+; SM60-NEXT: $L__BB167_11: // %atomicrmw.start47
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd39;
+; SM60-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
+; SM60-NEXT: sub.s64 %rd28, %rd14, %rd2;
+; SM60-NEXT: selp.b64 %rd29, %rd28, %rd14, %p11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM60-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
+; SM60-NEXT: @%p12 bra $L__BB167_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end46
+; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
+; SM60-NEXT: $L__BB167_13: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
+; SM60-NEXT: sub.s64 %rd30, %rd40, %rd11;
+; SM60-NEXT: selp.b64 %rd31, %rd30, %rd40, %p13;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM60-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
+; SM60-NEXT: mov.b64 %rd40, %rd15;
+; SM60-NEXT: @%p14 bra $L__BB167_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end57
+; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
+; SM60-NEXT: $L__BB167_15: // %atomicrmw.start64
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
+; SM60-NEXT: sub.s64 %rd32, %rd41, %rd12;
+; SM60-NEXT: selp.b64 %rd33, %rd32, %rd41, %p15;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM60-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
+; SM60-NEXT: mov.b64 %rd41, %rd16;
+; SM60-NEXT: @%p16 bra $L__BB167_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end63
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<22>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_sat_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r21, [%rd1];
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM60-NEXT: $L__BB168_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r21, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM60-NEXT: sub.s16 %rs9, %rs6, %rs8;
+; SM60-NEXT: setp.gt.u16 %p1, %rs9, %rs6;
+; SM60-NEXT: selp.b16 %rs10, -1, 0, %p1;
+; SM60-NEXT: sub.s16 %rs11, %rs5, %rs7;
+; SM60-NEXT: setp.gt.u16 %p2, %rs11, %rs5;
+; SM60-NEXT: selp.b16 %rs12, -1, 0, %p2;
+; SM60-NEXT: mov.b32 %r12, {%rs12, %rs10};
+; SM60-NEXT: xor.b32 %r13, %r12, -1;
+; SM60-NEXT: mov.b32 %r14, {%rs11, %rs9};
+; SM60-NEXT: and.b32 %r15, %r14, %r13;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r15;
+; SM60-NEXT: shl.b16 %rs15, %rs14, 8;
+; SM60-NEXT: or.b16 %rs16, %rs13, %rs15;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM60-NEXT: shl.b32 %r17, %r16, %r1;
+; SM60-NEXT: and.b32 %r18, %r21, %r2;
+; SM60-NEXT: or.b32 %r19, %r18, %r17;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r21, %r19;
+; SM60-NEXT: setp.ne.b32 %p3, %r3, %r21;
+; SM60-NEXT: mov.b32 %r21, %r3;
+; SM60-NEXT: @%p3 bra $L__BB168_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r20, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r20;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r13;
+; SM60-NEXT: $L__BB169_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM60-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r7;
+; SM60-NEXT: max.u16 %rs7, %rs6, %rs5;
+; SM60-NEXT: sub.s16 %rs8, %rs7, %rs5;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs8;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r11;
+; SM60-NEXT: max.u16 %rs11, %rs10, %rs9;
+; SM60-NEXT: sub.s16 %rs12, %rs11, %rs9;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r14;
+; SM60-NEXT: max.u16 %rs15, %rs14, %rs13;
+; SM60-NEXT: sub.s16 %rs16, %rs15, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs16;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB169_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<33>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB170_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM60-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r7;
+; SM60-NEXT: max.u16 %rs7, %rs6, %rs5;
+; SM60-NEXT: sub.s16 %rs8, %rs7, %rs5;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs8;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r11;
+; SM60-NEXT: max.u16 %rs11, %rs10, %rs9;
+; SM60-NEXT: sub.s16 %rs12, %rs11, %rs9;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r14;
+; SM60-NEXT: max.u16 %rs15, %rs14, %rs13;
+; SM60-NEXT: sub.s16 %rs16, %rs15, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs16;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs18, %r19;
+; SM60-NEXT: max.u16 %rs19, %rs18, %rs17;
+; SM60-NEXT: sub.s16 %rs20, %rs19, %rs17;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM60-NEXT: cvt.u16.u32 %rs21, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r22;
+; SM60-NEXT: max.u16 %rs23, %rs22, %rs21;
+; SM60-NEXT: sub.s16 %rs24, %rs23, %rs21;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs24;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs25, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs26, %r26;
+; SM60-NEXT: max.u16 %rs27, %rs26, %rs25;
+; SM60-NEXT: sub.s16 %rs28, %rs27, %rs25;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs28;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs29, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs30, %r29;
+; SM60-NEXT: max.u16 %rs31, %rs30, %rs29;
+; SM60-NEXT: sub.s16 %rs32, %rs31, %rs29;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs32;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB170_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r7, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB171_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r7;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: setp.gt.u16 %p1, %rs5, %rs4;
+; SM60-NEXT: selp.b16 %rs7, -1, 0, %p1;
+; SM60-NEXT: setp.gt.u16 %p2, %rs6, %rs3;
+; SM60-NEXT: selp.b16 %rs8, -1, 0, %p2;
+; SM60-NEXT: mov.b32 %r4, {%rs8, %rs7};
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: and.b32 %r6, %r3, %r5;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r7;
+; SM60-NEXT: mov.b32 %r7, %r1;
+; SM60-NEXT: @%p3 bra $L__BB171_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM60-NEXT: $L__BB172_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r11;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: setp.gt.u16 %p1, %rs5, %rs4;
+; SM60-NEXT: selp.b16 %rs7, -1, 0, %p1;
+; SM60-NEXT: setp.gt.u16 %p2, %rs6, %rs3;
+; SM60-NEXT: selp.b16 %rs8, -1, 0, %p2;
+; SM60-NEXT: mov.b32 %r4, {%rs8, %rs7};
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: and.b32 %r6, %r3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r6;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r12;
+; SM60-NEXT: sub.s16 %rs13, %rs12, %rs10;
+; SM60-NEXT: sub.s16 %rs14, %rs11, %rs9;
+; SM60-NEXT: mov.b32 %r7, {%rs14, %rs13};
+; SM60-NEXT: setp.gt.u16 %p3, %rs13, %rs12;
+; SM60-NEXT: selp.b16 %rs15, -1, 0, %p3;
+; SM60-NEXT: setp.gt.u16 %p4, %rs14, %rs11;
+; SM60-NEXT: selp.b16 %rs16, -1, 0, %p4;
+; SM60-NEXT: mov.b32 %r8, {%rs16, %rs15};
+; SM60-NEXT: xor.b32 %r9, %r8, -1;
+; SM60-NEXT: and.b32 %r10, %r7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r12;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB172_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<33>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM60-NEXT: $L__BB173_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r21;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: setp.gt.u16 %p1, %rs5, %rs4;
+; SM60-NEXT: selp.b16 %rs7, -1, 0, %p1;
+; SM60-NEXT: setp.gt.u16 %p2, %rs6, %rs3;
+; SM60-NEXT: selp.b16 %rs8, -1, 0, %p2;
+; SM60-NEXT: mov.b32 %r6, {%rs8, %rs7};
+; SM60-NEXT: xor.b32 %r7, %r6, -1;
+; SM60-NEXT: and.b32 %r8, %r5, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r8;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r22;
+; SM60-NEXT: sub.s16 %rs13, %rs12, %rs10;
+; SM60-NEXT: sub.s16 %rs14, %rs11, %rs9;
+; SM60-NEXT: mov.b32 %r9, {%rs14, %rs13};
+; SM60-NEXT: setp.gt.u16 %p3, %rs13, %rs12;
+; SM60-NEXT: selp.b16 %rs15, -1, 0, %p3;
+; SM60-NEXT: setp.gt.u16 %p4, %rs14, %rs11;
+; SM60-NEXT: selp.b16 %rs16, -1, 0, %p4;
+; SM60-NEXT: mov.b32 %r10, {%rs16, %rs15};
+; SM60-NEXT: xor.b32 %r11, %r10, -1;
+; SM60-NEXT: and.b32 %r12, %r9, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r22;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB173_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd12;
+; SM60-NEXT: $L__BB173_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r23;
+; SM60-NEXT: sub.s16 %rs21, %rs20, %rs18;
+; SM60-NEXT: sub.s16 %rs22, %rs19, %rs17;
+; SM60-NEXT: mov.b32 %r13, {%rs22, %rs21};
+; SM60-NEXT: setp.gt.u16 %p6, %rs21, %rs20;
+; SM60-NEXT: selp.b16 %rs23, -1, 0, %p6;
+; SM60-NEXT: setp.gt.u16 %p7, %rs22, %rs19;
+; SM60-NEXT: selp.b16 %rs24, -1, 0, %p7;
+; SM60-NEXT: mov.b32 %r14, {%rs24, %rs23};
+; SM60-NEXT: xor.b32 %r15, %r14, -1;
+; SM60-NEXT: and.b32 %r16, %r13, %r15;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r16;
+; SM60-NEXT: mov.b32 {%rs25, %rs26}, %r4;
+; SM60-NEXT: mov.b32 {%rs27, %rs28}, %r24;
+; SM60-NEXT: sub.s16 %rs29, %rs28, %rs26;
+; SM60-NEXT: sub.s16 %rs30, %rs27, %rs25;
+; SM60-NEXT: mov.b32 %r17, {%rs30, %rs29};
+; SM60-NEXT: setp.gt.u16 %p8, %rs29, %rs28;
+; SM60-NEXT: selp.b16 %rs31, -1, 0, %p8;
+; SM60-NEXT: setp.gt.u16 %p9, %rs30, %rs27;
+; SM60-NEXT: selp.b16 %rs32, -1, 0, %p9;
+; SM60-NEXT: mov.b32 %r18, {%rs32, %rs31};
+; SM60-NEXT: xor.b32 %r19, %r18, -1;
+; SM60-NEXT: and.b32 %r20, %r17, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r20;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r24;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB173_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB174_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r3, %r7, %r1;
+; SM60-NEXT: sub.s32 %r4, %r3, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r4;
+; SM60-NEXT: max.u32 %r5, %r8, %r2;
+; SM60-NEXT: sub.s32 %r6, %r5, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB174_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM60-NEXT: $L__BB175_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r5, %r13, %r1;
+; SM60-NEXT: sub.s32 %r6, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r6;
+; SM60-NEXT: max.u32 %r7, %r14, %r2;
+; SM60-NEXT: sub.s32 %r8, %r7, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB175_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM60-NEXT: $L__BB175_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r9, %r15, %r3;
+; SM60-NEXT: sub.s32 %r10, %r9, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r10;
+; SM60-NEXT: max.u32 %r11, %r16, %r4;
+; SM60-NEXT: sub.s32 %r12, %r11, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB175_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<33>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_sat_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM60-NEXT: $L__BB176_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r9, %r25, %r5;
+; SM60-NEXT: sub.s32 %r10, %r9, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r10;
+; SM60-NEXT: max.u32 %r11, %r26, %r6;
+; SM60-NEXT: sub.s32 %r12, %r11, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB176_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM60-NEXT: $L__BB176_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r13, %r27, %r7;
+; SM60-NEXT: sub.s32 %r14, %r13, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r14;
+; SM60-NEXT: max.u32 %r15, %r28, %r8;
+; SM60-NEXT: sub.s32 %r16, %r15, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r16;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB176_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd22;
+; SM60-NEXT: $L__BB176_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r17, %r29, %r1;
+; SM60-NEXT: sub.s32 %r18, %r17, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r18;
+; SM60-NEXT: max.u32 %r19, %r30, %r2;
+; SM60-NEXT: sub.s32 %r20, %r19, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r20;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r29;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r30;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM60-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd31;
+; SM60-NEXT: @%p3 bra $L__BB176_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r31, %rd32;
+; SM60-NEXT: $L__BB176_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r21, %r31, %r3;
+; SM60-NEXT: sub.s32 %r22, %r21, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r22;
+; SM60-NEXT: max.u32 %r23, %r32, %r4;
+; SM60-NEXT: sub.s32 %r24, %r23, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r24;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r31;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r32;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM60-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r31, %rd41;
+; SM60-NEXT: @%p4 bra $L__BB176_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [usub_sat_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
+; SM60-NEXT: $L__BB177_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd6, %rd10, %rd1;
+; SM60-NEXT: sub.s64 %rd7, %rd6, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
+; SM60-NEXT: mov.b64 %rd10, %rd3;
+; SM60-NEXT: @%p1 bra $L__BB177_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
+; SM60-NEXT: $L__BB177_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd8, %rd11, %rd2;
+; SM60-NEXT: sub.s64 %rd9, %rd8, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
+; SM60-NEXT: mov.b64 %rd11, %rd4;
+; SM60-NEXT: @%p2 bra $L__BB177_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [usub_sat_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
+; SM60-NEXT: $L__BB178_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd18;
+; SM60-NEXT: max.u64 %rd10, %rd5, %rd3;
+; SM60-NEXT: sub.s64 %rd11, %rd10, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
+; SM60-NEXT: @%p1 bra $L__BB178_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
+; SM60-NEXT: $L__BB178_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd19;
+; SM60-NEXT: max.u64 %rd12, %rd6, %rd4;
+; SM60-NEXT: sub.s64 %rd13, %rd12, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM60-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
+; SM60-NEXT: @%p2 bra $L__BB178_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
+; SM60-NEXT: $L__BB178_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd14, %rd20, %rd1;
+; SM60-NEXT: sub.s64 %rd15, %rd14, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
+; SM60-NEXT: mov.b64 %rd20, %rd7;
+; SM60-NEXT: @%p3 bra $L__BB178_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
+; SM60-NEXT: $L__BB178_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd16, %rd21, %rd2;
+; SM60-NEXT: sub.s64 %rd17, %rd16, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
+; SM60-NEXT: mov.b64 %rd21, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB178_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [usub_sat_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [usub_sat_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
+; SM60-NEXT: $L__BB179_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd34;
+; SM60-NEXT: max.u64 %rd18, %rd7, %rd5;
+; SM60-NEXT: sub.s64 %rd19, %rd18, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM60-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
+; SM60-NEXT: @%p1 bra $L__BB179_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
+; SM60-NEXT: $L__BB179_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd35;
+; SM60-NEXT: max.u64 %rd20, %rd8, %rd6;
+; SM60-NEXT: sub.s64 %rd21, %rd20, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM60-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
+; SM60-NEXT: @%p2 bra $L__BB179_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
+; SM60-NEXT: $L__BB179_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd36;
+; SM60-NEXT: max.u64 %rd22, %rd9, %rd3;
+; SM60-NEXT: sub.s64 %rd23, %rd22, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM60-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
+; SM60-NEXT: @%p3 bra $L__BB179_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
+; SM60-NEXT: $L__BB179_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd37;
+; SM60-NEXT: max.u64 %rd24, %rd10, %rd4;
+; SM60-NEXT: sub.s64 %rd25, %rd24, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM60-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
+; SM60-NEXT: @%p4 bra $L__BB179_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
+; SM60-NEXT: $L__BB179_9: // %atomicrmw.start41
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd38;
+; SM60-NEXT: max.u64 %rd26, %rd13, %rd1;
+; SM60-NEXT: sub.s64 %rd27, %rd26, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM60-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
+; SM60-NEXT: @%p5 bra $L__BB179_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end40
+; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
+; SM60-NEXT: $L__BB179_11: // %atomicrmw.start47
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd39;
+; SM60-NEXT: max.u64 %rd28, %rd14, %rd2;
+; SM60-NEXT: sub.s64 %rd29, %rd28, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM60-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
+; SM60-NEXT: @%p6 bra $L__BB179_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end46
+; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
+; SM60-NEXT: $L__BB179_13: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd30, %rd40, %rd11;
+; SM60-NEXT: sub.s64 %rd31, %rd30, %rd11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
+; SM60-NEXT: mov.b64 %rd40, %rd15;
+; SM60-NEXT: @%p7 bra $L__BB179_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end57
+; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
+; SM60-NEXT: $L__BB179_15: // %atomicrmw.start64
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd32, %rd41, %rd12;
+; SM60-NEXT: sub.s64 %rd33, %rd32, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
+; SM60-NEXT: mov.b64 %rd41, %rd16;
+; SM60-NEXT: @%p8 bra $L__BB179_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end63
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM60-LABEL: fadd_acq_rel_v2f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.f32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM60-LABEL: fadd_acq_rel_v4f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.f32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.f32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.f32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM60-LABEL: fadd_acq_rel_v8f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.f32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.f32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.f32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.f32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.f32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.f32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM60-LABEL: fsub_acq_rel_v2f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB183_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: sub.rn.f32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB183_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM60-LABEL: fsub_acq_rel_v4f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB184_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: sub.rn.f32 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB184_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB184_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: sub.rn.f32 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB184_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM60-LABEL: fsub_acq_rel_v8f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB185_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r9, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: sub.rn.f32 %r10, %r18, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB185_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM60-NEXT: $L__BB185_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r11, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: sub.rn.f32 %r12, %r20, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB185_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM60-NEXT: $L__BB185_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r13, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r13;
+; SM60-NEXT: sub.rn.f32 %r14, %r22, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM60-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM60-NEXT: @%p3 bra $L__BB185_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM60-NEXT: $L__BB185_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r15, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM60-NEXT: sub.rn.f32 %r16, %r24, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM60-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM60-NEXT: @%p4 bra $L__BB185_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM60-LABEL: fmin_acq_rel_v2f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v2f32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB186_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: min.f32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB186_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM60-LABEL: fmin_acq_rel_v4f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v4f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB187_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: min.f32 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB187_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB187_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: min.f32 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB187_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM60-LABEL: fmin_acq_rel_v8f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmin_acq_rel_v8f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB188_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r9, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: min.f32 %r10, %r18, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB188_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM60-NEXT: $L__BB188_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r11, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: min.f32 %r12, %r20, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB188_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM60-NEXT: $L__BB188_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r13, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r13;
+; SM60-NEXT: min.f32 %r14, %r22, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM60-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM60-NEXT: @%p3 bra $L__BB188_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM60-NEXT: $L__BB188_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r15, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM60-NEXT: min.f32 %r16, %r24, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM60-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM60-NEXT: @%p4 bra $L__BB188_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM60-LABEL: fmax_acq_rel_v2f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v2f32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB189_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: max.f32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB189_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM60-LABEL: fmax_acq_rel_v4f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v4f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB190_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: max.f32 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB190_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB190_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: max.f32 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB190_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM60-LABEL: fmax_acq_rel_v8f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmax_acq_rel_v8f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB191_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r9, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: max.f32 %r10, %r18, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB191_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM60-NEXT: $L__BB191_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r11, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: max.f32 %r12, %r20, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB191_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM60-NEXT: $L__BB191_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r13, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r13;
+; SM60-NEXT: max.f32 %r14, %r22, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM60-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM60-NEXT: @%p3 bra $L__BB191_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM60-NEXT: $L__BB191_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r15, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM60-NEXT: max.f32 %r16, %r24, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM60-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM60-NEXT: @%p4 bra $L__BB191_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM60-LABEL: fminimum_acq_rel_v2f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v2f32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM60-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
+; SM60-NEXT: $L__BB192_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r13, %r1;
+; SM60-NEXT: min.f32 %r3, %r13, %r1;
+; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r13, -2147483648;
+; SM60-NEXT: selp.f32 %r5, %r13, %r4, %p2;
+; SM60-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: setp.nan.f32 %p5, %r14, %r2;
+; SM60-NEXT: min.f32 %r8, %r14, %r2;
+; SM60-NEXT: selp.f32 %r9, 0f7FC00000, %r8, %p5;
+; SM60-NEXT: setp.eq.b32 %p6, %r14, -2147483648;
+; SM60-NEXT: selp.f32 %r10, %r14, %r9, %p6;
+; SM60-NEXT: selp.f32 %r11, %r2, %r10, %p7;
+; SM60-NEXT: setp.eq.f32 %p8, %r9, 0f00000000;
+; SM60-NEXT: selp.f32 %r12, %r11, %r9, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB192_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM60-LABEL: fminimum_acq_rel_v4f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<19>;
+; SM60-NEXT: .reg .b32 %r<29>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM60-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
+; SM60-NEXT: $L__BB193_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r25, %r1;
+; SM60-NEXT: min.f32 %r5, %r25, %r1;
+; SM60-NEXT: selp.f32 %r6, 0f7FC00000, %r5, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r25, -2147483648;
+; SM60-NEXT: selp.f32 %r7, %r25, %r6, %p2;
+; SM60-NEXT: selp.f32 %r8, %r1, %r7, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r6, 0f00000000;
+; SM60-NEXT: selp.f32 %r9, %r8, %r6, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: setp.nan.f32 %p5, %r26, %r2;
+; SM60-NEXT: min.f32 %r10, %r26, %r2;
+; SM60-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p5;
+; SM60-NEXT: setp.eq.b32 %p6, %r26, -2147483648;
+; SM60-NEXT: selp.f32 %r12, %r26, %r11, %p6;
+; SM60-NEXT: selp.f32 %r13, %r2, %r12, %p7;
+; SM60-NEXT: setp.eq.f32 %p8, %r11, 0f00000000;
+; SM60-NEXT: selp.f32 %r14, %r13, %r11, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB193_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM60-NEXT: setp.eq.b32 %p12, %r3, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p16, %r4, -2147483648;
+; SM60-NEXT: $L__BB193_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p10, %r27, %r3;
+; SM60-NEXT: min.f32 %r15, %r27, %r3;
+; SM60-NEXT: selp.f32 %r16, 0f7FC00000, %r15, %p10;
+; SM60-NEXT: setp.eq.b32 %p11, %r27, -2147483648;
+; SM60-NEXT: selp.f32 %r17, %r27, %r16, %p11;
+; SM60-NEXT: selp.f32 %r18, %r3, %r17, %p12;
+; SM60-NEXT: setp.eq.f32 %p13, %r16, 0f00000000;
+; SM60-NEXT: selp.f32 %r19, %r18, %r16, %p13;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r19;
+; SM60-NEXT: setp.nan.f32 %p14, %r28, %r4;
+; SM60-NEXT: min.f32 %r20, %r28, %r4;
+; SM60-NEXT: selp.f32 %r21, 0f7FC00000, %r20, %p14;
+; SM60-NEXT: setp.eq.b32 %p15, %r28, -2147483648;
+; SM60-NEXT: selp.f32 %r22, %r28, %r21, %p15;
+; SM60-NEXT: selp.f32 %r23, %r4, %r22, %p16;
+; SM60-NEXT: setp.eq.f32 %p17, %r21, 0f00000000;
+; SM60-NEXT: selp.f32 %r24, %r23, %r21, %p17;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r24;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM60-NEXT: @%p18 bra $L__BB193_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM60-LABEL: fminimum_acq_rel_v8f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<37>;
+; SM60-NEXT: .reg .b32 %r<57>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM60-NEXT: setp.eq.b32 %p3, %r5, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p7, %r6, -2147483648;
+; SM60-NEXT: $L__BB194_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r49, %r5;
+; SM60-NEXT: min.f32 %r9, %r49, %r5;
+; SM60-NEXT: selp.f32 %r10, 0f7FC00000, %r9, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r49, -2147483648;
+; SM60-NEXT: selp.f32 %r11, %r49, %r10, %p2;
+; SM60-NEXT: selp.f32 %r12, %r5, %r11, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r10, 0f00000000;
+; SM60-NEXT: selp.f32 %r13, %r12, %r10, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM60-NEXT: setp.nan.f32 %p5, %r50, %r6;
+; SM60-NEXT: min.f32 %r14, %r50, %r6;
+; SM60-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p5;
+; SM60-NEXT: setp.eq.b32 %p6, %r50, -2147483648;
+; SM60-NEXT: selp.f32 %r16, %r50, %r15, %p6;
+; SM60-NEXT: selp.f32 %r17, %r6, %r16, %p7;
+; SM60-NEXT: setp.eq.f32 %p8, %r15, 0f00000000;
+; SM60-NEXT: selp.f32 %r18, %r17, %r15, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r49;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r50;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB194_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r51, %rd12;
+; SM60-NEXT: setp.eq.b32 %p12, %r7, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p16, %r8, -2147483648;
+; SM60-NEXT: $L__BB194_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p10, %r51, %r7;
+; SM60-NEXT: min.f32 %r19, %r51, %r7;
+; SM60-NEXT: selp.f32 %r20, 0f7FC00000, %r19, %p10;
+; SM60-NEXT: setp.eq.b32 %p11, %r51, -2147483648;
+; SM60-NEXT: selp.f32 %r21, %r51, %r20, %p11;
+; SM60-NEXT: selp.f32 %r22, %r7, %r21, %p12;
+; SM60-NEXT: setp.eq.f32 %p13, %r20, 0f00000000;
+; SM60-NEXT: selp.f32 %r23, %r22, %r20, %p13;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r23;
+; SM60-NEXT: setp.nan.f32 %p14, %r52, %r8;
+; SM60-NEXT: min.f32 %r24, %r52, %r8;
+; SM60-NEXT: selp.f32 %r25, 0f7FC00000, %r24, %p14;
+; SM60-NEXT: setp.eq.b32 %p15, %r52, -2147483648;
+; SM60-NEXT: selp.f32 %r26, %r52, %r25, %p15;
+; SM60-NEXT: selp.f32 %r27, %r8, %r26, %p16;
+; SM60-NEXT: setp.eq.f32 %p17, %r25, 0f00000000;
+; SM60-NEXT: selp.f32 %r28, %r27, %r25, %p17;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r51;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r52;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r51, %rd21;
+; SM60-NEXT: @%p18 bra $L__BB194_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r53, %rd22;
+; SM60-NEXT: setp.eq.b32 %p21, %r1, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p25, %r2, -2147483648;
+; SM60-NEXT: $L__BB194_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p19, %r53, %r1;
+; SM60-NEXT: min.f32 %r29, %r53, %r1;
+; SM60-NEXT: selp.f32 %r30, 0f7FC00000, %r29, %p19;
+; SM60-NEXT: setp.eq.b32 %p20, %r53, -2147483648;
+; SM60-NEXT: selp.f32 %r31, %r53, %r30, %p20;
+; SM60-NEXT: selp.f32 %r32, %r1, %r31, %p21;
+; SM60-NEXT: setp.eq.f32 %p22, %r30, 0f00000000;
+; SM60-NEXT: selp.f32 %r33, %r32, %r30, %p22;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r33;
+; SM60-NEXT: setp.nan.f32 %p23, %r54, %r2;
+; SM60-NEXT: min.f32 %r34, %r54, %r2;
+; SM60-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
+; SM60-NEXT: setp.eq.b32 %p24, %r54, -2147483648;
+; SM60-NEXT: selp.f32 %r36, %r54, %r35, %p24;
+; SM60-NEXT: selp.f32 %r37, %r2, %r36, %p25;
+; SM60-NEXT: setp.eq.f32 %p26, %r35, 0f00000000;
+; SM60-NEXT: selp.f32 %r38, %r37, %r35, %p26;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r38;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r53;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r54;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM60-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r53, %rd31;
+; SM60-NEXT: @%p27 bra $L__BB194_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r55, %rd32;
+; SM60-NEXT: setp.eq.b32 %p30, %r3, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p34, %r4, -2147483648;
+; SM60-NEXT: $L__BB194_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p28, %r55, %r3;
+; SM60-NEXT: min.f32 %r39, %r55, %r3;
+; SM60-NEXT: selp.f32 %r40, 0f7FC00000, %r39, %p28;
+; SM60-NEXT: setp.eq.b32 %p29, %r55, -2147483648;
+; SM60-NEXT: selp.f32 %r41, %r55, %r40, %p29;
+; SM60-NEXT: selp.f32 %r42, %r3, %r41, %p30;
+; SM60-NEXT: setp.eq.f32 %p31, %r40, 0f00000000;
+; SM60-NEXT: selp.f32 %r43, %r42, %r40, %p31;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r43;
+; SM60-NEXT: setp.nan.f32 %p32, %r56, %r4;
+; SM60-NEXT: min.f32 %r44, %r56, %r4;
+; SM60-NEXT: selp.f32 %r45, 0f7FC00000, %r44, %p32;
+; SM60-NEXT: setp.eq.b32 %p33, %r56, -2147483648;
+; SM60-NEXT: selp.f32 %r46, %r56, %r45, %p33;
+; SM60-NEXT: selp.f32 %r47, %r4, %r46, %p34;
+; SM60-NEXT: setp.eq.f32 %p35, %r45, 0f00000000;
+; SM60-NEXT: selp.f32 %r48, %r47, %r45, %p35;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r48;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r55;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r56;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM60-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r55, %rd41;
+; SM60-NEXT: @%p36 bra $L__BB194_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r49, %r50, %r51, %r52};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v2f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v2f32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM60-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM60-NEXT: setp.eq.b32 %p7, %r2, 0;
+; SM60-NEXT: $L__BB195_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r13, %r1;
+; SM60-NEXT: max.f32 %r3, %r13, %r1;
+; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r13, 0;
+; SM60-NEXT: selp.f32 %r5, %r13, %r4, %p2;
+; SM60-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: setp.nan.f32 %p5, %r14, %r2;
+; SM60-NEXT: max.f32 %r8, %r14, %r2;
+; SM60-NEXT: selp.f32 %r9, 0f7FC00000, %r8, %p5;
+; SM60-NEXT: setp.eq.b32 %p6, %r14, 0;
+; SM60-NEXT: selp.f32 %r10, %r14, %r9, %p6;
+; SM60-NEXT: selp.f32 %r11, %r2, %r10, %p7;
+; SM60-NEXT: setp.eq.f32 %p8, %r9, 0f00000000;
+; SM60-NEXT: selp.f32 %r12, %r11, %r9, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB195_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v4f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<19>;
+; SM60-NEXT: .reg .b32 %r<29>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM60-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM60-NEXT: setp.eq.b32 %p7, %r2, 0;
+; SM60-NEXT: $L__BB196_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r25, %r1;
+; SM60-NEXT: max.f32 %r5, %r25, %r1;
+; SM60-NEXT: selp.f32 %r6, 0f7FC00000, %r5, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r25, 0;
+; SM60-NEXT: selp.f32 %r7, %r25, %r6, %p2;
+; SM60-NEXT: selp.f32 %r8, %r1, %r7, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r6, 0f00000000;
+; SM60-NEXT: selp.f32 %r9, %r8, %r6, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: setp.nan.f32 %p5, %r26, %r2;
+; SM60-NEXT: max.f32 %r10, %r26, %r2;
+; SM60-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p5;
+; SM60-NEXT: setp.eq.b32 %p6, %r26, 0;
+; SM60-NEXT: selp.f32 %r12, %r26, %r11, %p6;
+; SM60-NEXT: selp.f32 %r13, %r2, %r12, %p7;
+; SM60-NEXT: setp.eq.f32 %p8, %r11, 0f00000000;
+; SM60-NEXT: selp.f32 %r14, %r13, %r11, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB196_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM60-NEXT: setp.eq.b32 %p12, %r3, 0;
+; SM60-NEXT: setp.eq.b32 %p16, %r4, 0;
+; SM60-NEXT: $L__BB196_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p10, %r27, %r3;
+; SM60-NEXT: max.f32 %r15, %r27, %r3;
+; SM60-NEXT: selp.f32 %r16, 0f7FC00000, %r15, %p10;
+; SM60-NEXT: setp.eq.b32 %p11, %r27, 0;
+; SM60-NEXT: selp.f32 %r17, %r27, %r16, %p11;
+; SM60-NEXT: selp.f32 %r18, %r3, %r17, %p12;
+; SM60-NEXT: setp.eq.f32 %p13, %r16, 0f00000000;
+; SM60-NEXT: selp.f32 %r19, %r18, %r16, %p13;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r19;
+; SM60-NEXT: setp.nan.f32 %p14, %r28, %r4;
+; SM60-NEXT: max.f32 %r20, %r28, %r4;
+; SM60-NEXT: selp.f32 %r21, 0f7FC00000, %r20, %p14;
+; SM60-NEXT: setp.eq.b32 %p15, %r28, 0;
+; SM60-NEXT: selp.f32 %r22, %r28, %r21, %p15;
+; SM60-NEXT: selp.f32 %r23, %r4, %r22, %p16;
+; SM60-NEXT: setp.eq.f32 %p17, %r21, 0f00000000;
+; SM60-NEXT: selp.f32 %r24, %r23, %r21, %p17;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r24;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM60-NEXT: @%p18 bra $L__BB196_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v8f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<37>;
+; SM60-NEXT: .reg .b32 %r<57>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM60-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM60-NEXT: setp.eq.b32 %p7, %r6, 0;
+; SM60-NEXT: $L__BB197_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r49, %r5;
+; SM60-NEXT: max.f32 %r9, %r49, %r5;
+; SM60-NEXT: selp.f32 %r10, 0f7FC00000, %r9, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r49, 0;
+; SM60-NEXT: selp.f32 %r11, %r49, %r10, %p2;
+; SM60-NEXT: selp.f32 %r12, %r5, %r11, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r10, 0f00000000;
+; SM60-NEXT: selp.f32 %r13, %r12, %r10, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM60-NEXT: setp.nan.f32 %p5, %r50, %r6;
+; SM60-NEXT: max.f32 %r14, %r50, %r6;
+; SM60-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p5;
+; SM60-NEXT: setp.eq.b32 %p6, %r50, 0;
+; SM60-NEXT: selp.f32 %r16, %r50, %r15, %p6;
+; SM60-NEXT: selp.f32 %r17, %r6, %r16, %p7;
+; SM60-NEXT: setp.eq.f32 %p8, %r15, 0f00000000;
+; SM60-NEXT: selp.f32 %r18, %r17, %r15, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r49;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r50;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB197_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r51, %rd12;
+; SM60-NEXT: setp.eq.b32 %p12, %r7, 0;
+; SM60-NEXT: setp.eq.b32 %p16, %r8, 0;
+; SM60-NEXT: $L__BB197_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p10, %r51, %r7;
+; SM60-NEXT: max.f32 %r19, %r51, %r7;
+; SM60-NEXT: selp.f32 %r20, 0f7FC00000, %r19, %p10;
+; SM60-NEXT: setp.eq.b32 %p11, %r51, 0;
+; SM60-NEXT: selp.f32 %r21, %r51, %r20, %p11;
+; SM60-NEXT: selp.f32 %r22, %r7, %r21, %p12;
+; SM60-NEXT: setp.eq.f32 %p13, %r20, 0f00000000;
+; SM60-NEXT: selp.f32 %r23, %r22, %r20, %p13;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r23;
+; SM60-NEXT: setp.nan.f32 %p14, %r52, %r8;
+; SM60-NEXT: max.f32 %r24, %r52, %r8;
+; SM60-NEXT: selp.f32 %r25, 0f7FC00000, %r24, %p14;
+; SM60-NEXT: setp.eq.b32 %p15, %r52, 0;
+; SM60-NEXT: selp.f32 %r26, %r52, %r25, %p15;
+; SM60-NEXT: selp.f32 %r27, %r8, %r26, %p16;
+; SM60-NEXT: setp.eq.f32 %p17, %r25, 0f00000000;
+; SM60-NEXT: selp.f32 %r28, %r27, %r25, %p17;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r51;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r52;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r51, %rd21;
+; SM60-NEXT: @%p18 bra $L__BB197_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r53, %rd22;
+; SM60-NEXT: setp.eq.b32 %p21, %r1, 0;
+; SM60-NEXT: setp.eq.b32 %p25, %r2, 0;
+; SM60-NEXT: $L__BB197_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p19, %r53, %r1;
+; SM60-NEXT: max.f32 %r29, %r53, %r1;
+; SM60-NEXT: selp.f32 %r30, 0f7FC00000, %r29, %p19;
+; SM60-NEXT: setp.eq.b32 %p20, %r53, 0;
+; SM60-NEXT: selp.f32 %r31, %r53, %r30, %p20;
+; SM60-NEXT: selp.f32 %r32, %r1, %r31, %p21;
+; SM60-NEXT: setp.eq.f32 %p22, %r30, 0f00000000;
+; SM60-NEXT: selp.f32 %r33, %r32, %r30, %p22;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r33;
+; SM60-NEXT: setp.nan.f32 %p23, %r54, %r2;
+; SM60-NEXT: max.f32 %r34, %r54, %r2;
+; SM60-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
+; SM60-NEXT: setp.eq.b32 %p24, %r54, 0;
+; SM60-NEXT: selp.f32 %r36, %r54, %r35, %p24;
+; SM60-NEXT: selp.f32 %r37, %r2, %r36, %p25;
+; SM60-NEXT: setp.eq.f32 %p26, %r35, 0f00000000;
+; SM60-NEXT: selp.f32 %r38, %r37, %r35, %p26;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r38;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r53;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r54;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM60-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r53, %rd31;
+; SM60-NEXT: @%p27 bra $L__BB197_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r55, %rd32;
+; SM60-NEXT: setp.eq.b32 %p30, %r3, 0;
+; SM60-NEXT: setp.eq.b32 %p34, %r4, 0;
+; SM60-NEXT: $L__BB197_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p28, %r55, %r3;
+; SM60-NEXT: max.f32 %r39, %r55, %r3;
+; SM60-NEXT: selp.f32 %r40, 0f7FC00000, %r39, %p28;
+; SM60-NEXT: setp.eq.b32 %p29, %r55, 0;
+; SM60-NEXT: selp.f32 %r41, %r55, %r40, %p29;
+; SM60-NEXT: selp.f32 %r42, %r3, %r41, %p30;
+; SM60-NEXT: setp.eq.f32 %p31, %r40, 0f00000000;
+; SM60-NEXT: selp.f32 %r43, %r42, %r40, %p31;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r43;
+; SM60-NEXT: setp.nan.f32 %p32, %r56, %r4;
+; SM60-NEXT: max.f32 %r44, %r56, %r4;
+; SM60-NEXT: selp.f32 %r45, 0f7FC00000, %r44, %p32;
+; SM60-NEXT: setp.eq.b32 %p33, %r56, 0;
+; SM60-NEXT: selp.f32 %r46, %r56, %r45, %p33;
+; SM60-NEXT: selp.f32 %r47, %r4, %r46, %p34;
+; SM60-NEXT: setp.eq.f32 %p35, %r45, 0f00000000;
+; SM60-NEXT: selp.f32 %r48, %r47, %r45, %p35;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r48;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r55;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r56;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM60-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r55, %rd41;
+; SM60-NEXT: @%p36 bra $L__BB197_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r49, %r50, %r51, %r52};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x double> @fadd_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM60-LABEL: fadd_acq_rel_v2f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.add.f64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fadd_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM60-LABEL: fadd_acq_rel_v4f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.add.f64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.add.f64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.add.f64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fadd_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM60-LABEL: fadd_acq_rel_v8f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [fadd_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [fadd_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.add.f64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.add.f64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.add.f64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.add.f64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.add.f64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.add.f64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.add.f64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM60-LABEL: fsub_acq_rel_v2f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [fsub_acq_rel_v2f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd5];
+; SM60-NEXT: $L__BB201_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd6, %rd8, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM60-NEXT: mov.b64 %rd8, %rd3;
+; SM60-NEXT: @%p1 bra $L__BB201_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd5+8];
+; SM60-NEXT: $L__BB201_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd7, %rd9, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
+; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
+; SM60-NEXT: mov.b64 %rd9, %rd4;
+; SM60-NEXT: @%p2 bra $L__BB201_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM60-LABEL: fsub_acq_rel_v4f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [fsub_acq_rel_v4f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd9];
+; SM60-NEXT: $L__BB202_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd14;
+; SM60-NEXT: sub.rn.f64 %rd10, %rd5, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
+; SM60-NEXT: @%p1 bra $L__BB202_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd9+8];
+; SM60-NEXT: $L__BB202_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd15;
+; SM60-NEXT: sub.rn.f64 %rd11, %rd6, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
+; SM60-NEXT: @%p2 bra $L__BB202_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd9+16];
+; SM60-NEXT: $L__BB202_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd12, %rd16, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM60-NEXT: mov.b64 %rd16, %rd7;
+; SM60-NEXT: @%p3 bra $L__BB202_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd9+24];
+; SM60-NEXT: $L__BB202_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd13, %rd17, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
+; SM60-NEXT: mov.b64 %rd17, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB202_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM60-LABEL: fsub_acq_rel_v8f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<34>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [fsub_acq_rel_v8f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fsub_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fsub_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd17];
+; SM60-NEXT: $L__BB203_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd26;
+; SM60-NEXT: sub.rn.f64 %rd18, %rd7, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM60-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
+; SM60-NEXT: @%p1 bra $L__BB203_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd17+8];
+; SM60-NEXT: $L__BB203_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd27;
+; SM60-NEXT: sub.rn.f64 %rd19, %rd8, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
+; SM60-NEXT: @%p2 bra $L__BB203_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd17+16];
+; SM60-NEXT: $L__BB203_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd28;
+; SM60-NEXT: sub.rn.f64 %rd20, %rd9, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM60-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
+; SM60-NEXT: @%p3 bra $L__BB203_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd17+24];
+; SM60-NEXT: $L__BB203_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd29;
+; SM60-NEXT: sub.rn.f64 %rd21, %rd10, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
+; SM60-NEXT: @%p4 bra $L__BB203_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd17+32];
+; SM60-NEXT: $L__BB203_9: // %atomicrmw.start41
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd30;
+; SM60-NEXT: sub.rn.f64 %rd22, %rd13, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
+; SM60-NEXT: @%p5 bra $L__BB203_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end40
+; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd17+40];
+; SM60-NEXT: $L__BB203_11: // %atomicrmw.start47
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd31;
+; SM60-NEXT: sub.rn.f64 %rd23, %rd14, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
+; SM60-NEXT: @%p6 bra $L__BB203_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end46
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd17+48];
+; SM60-NEXT: $L__BB203_13: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd24, %rd32, %rd11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM60-NEXT: mov.b64 %rd32, %rd15;
+; SM60-NEXT: @%p7 bra $L__BB203_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end57
+; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd17+56];
+; SM60-NEXT: $L__BB203_15: // %atomicrmw.start64
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd25, %rd33, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
+; SM60-NEXT: mov.b64 %rd33, %rd16;
+; SM60-NEXT: @%p8 bra $L__BB203_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end63
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM60-LABEL: fmin_acq_rel_v2f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [fmin_acq_rel_v2f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd5];
+; SM60-NEXT: $L__BB204_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd6, %rd8, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM60-NEXT: mov.b64 %rd8, %rd3;
+; SM60-NEXT: @%p1 bra $L__BB204_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd5+8];
+; SM60-NEXT: $L__BB204_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd7, %rd9, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
+; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
+; SM60-NEXT: mov.b64 %rd9, %rd4;
+; SM60-NEXT: @%p2 bra $L__BB204_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM60-LABEL: fmin_acq_rel_v4f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [fmin_acq_rel_v4f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd9];
+; SM60-NEXT: $L__BB205_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd14;
+; SM60-NEXT: min.f64 %rd10, %rd5, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
+; SM60-NEXT: @%p1 bra $L__BB205_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd9+8];
+; SM60-NEXT: $L__BB205_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd15;
+; SM60-NEXT: min.f64 %rd11, %rd6, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
+; SM60-NEXT: @%p2 bra $L__BB205_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd9+16];
+; SM60-NEXT: $L__BB205_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd12, %rd16, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM60-NEXT: mov.b64 %rd16, %rd7;
+; SM60-NEXT: @%p3 bra $L__BB205_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd9+24];
+; SM60-NEXT: $L__BB205_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd13, %rd17, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
+; SM60-NEXT: mov.b64 %rd17, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB205_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM60-LABEL: fmin_acq_rel_v8f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<34>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [fmin_acq_rel_v8f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmin_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmin_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd17];
+; SM60-NEXT: $L__BB206_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd26;
+; SM60-NEXT: min.f64 %rd18, %rd7, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM60-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
+; SM60-NEXT: @%p1 bra $L__BB206_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd17+8];
+; SM60-NEXT: $L__BB206_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd27;
+; SM60-NEXT: min.f64 %rd19, %rd8, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
+; SM60-NEXT: @%p2 bra $L__BB206_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd17+16];
+; SM60-NEXT: $L__BB206_5: // %atomicrmw.start19
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd28;
+; SM60-NEXT: min.f64 %rd20, %rd9, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM60-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
+; SM60-NEXT: @%p3 bra $L__BB206_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end18
+; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd17+24];
+; SM60-NEXT: $L__BB206_7: // %atomicrmw.start24
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd29;
+; SM60-NEXT: min.f64 %rd21, %rd10, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
+; SM60-NEXT: @%p4 bra $L__BB206_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end23
+; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd17+32];
+; SM60-NEXT: $L__BB206_9: // %atomicrmw.start38
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd30;
+; SM60-NEXT: min.f64 %rd22, %rd13, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
+; SM60-NEXT: @%p5 bra $L__BB206_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end37
+; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd17+40];
+; SM60-NEXT: $L__BB206_11: // %atomicrmw.start43
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd31;
+; SM60-NEXT: min.f64 %rd23, %rd14, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
+; SM60-NEXT: @%p6 bra $L__BB206_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end42
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd17+48];
+; SM60-NEXT: $L__BB206_13: // %atomicrmw.start53
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd24, %rd32, %rd11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM60-NEXT: mov.b64 %rd32, %rd15;
+; SM60-NEXT: @%p7 bra $L__BB206_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end52
+; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd17+56];
+; SM60-NEXT: $L__BB206_15: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd25, %rd33, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
+; SM60-NEXT: mov.b64 %rd33, %rd16;
+; SM60-NEXT: @%p8 bra $L__BB206_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end57
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM60-LABEL: fmax_acq_rel_v2f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [fmax_acq_rel_v2f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd5];
+; SM60-NEXT: $L__BB207_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd6, %rd8, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM60-NEXT: mov.b64 %rd8, %rd3;
+; SM60-NEXT: @%p1 bra $L__BB207_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd5+8];
+; SM60-NEXT: $L__BB207_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd7, %rd9, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
+; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
+; SM60-NEXT: mov.b64 %rd9, %rd4;
+; SM60-NEXT: @%p2 bra $L__BB207_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM60-LABEL: fmax_acq_rel_v4f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [fmax_acq_rel_v4f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd9];
+; SM60-NEXT: $L__BB208_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd14;
+; SM60-NEXT: max.f64 %rd10, %rd5, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
+; SM60-NEXT: @%p1 bra $L__BB208_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd9+8];
+; SM60-NEXT: $L__BB208_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd15;
+; SM60-NEXT: max.f64 %rd11, %rd6, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
+; SM60-NEXT: @%p2 bra $L__BB208_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd9+16];
+; SM60-NEXT: $L__BB208_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd12, %rd16, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM60-NEXT: mov.b64 %rd16, %rd7;
+; SM60-NEXT: @%p3 bra $L__BB208_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd9+24];
+; SM60-NEXT: $L__BB208_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd13, %rd17, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
+; SM60-NEXT: mov.b64 %rd17, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB208_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM60-LABEL: fmax_acq_rel_v8f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<34>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [fmax_acq_rel_v8f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmax_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmax_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd17];
+; SM60-NEXT: $L__BB209_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd26;
+; SM60-NEXT: max.f64 %rd18, %rd7, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM60-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
+; SM60-NEXT: @%p1 bra $L__BB209_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd17+8];
+; SM60-NEXT: $L__BB209_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd27;
+; SM60-NEXT: max.f64 %rd19, %rd8, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
+; SM60-NEXT: @%p2 bra $L__BB209_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd17+16];
+; SM60-NEXT: $L__BB209_5: // %atomicrmw.start19
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd28;
+; SM60-NEXT: max.f64 %rd20, %rd9, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM60-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
+; SM60-NEXT: @%p3 bra $L__BB209_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end18
+; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd17+24];
+; SM60-NEXT: $L__BB209_7: // %atomicrmw.start24
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd29;
+; SM60-NEXT: max.f64 %rd21, %rd10, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
+; SM60-NEXT: @%p4 bra $L__BB209_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end23
+; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd17+32];
+; SM60-NEXT: $L__BB209_9: // %atomicrmw.start38
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd30;
+; SM60-NEXT: max.f64 %rd22, %rd13, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
+; SM60-NEXT: @%p5 bra $L__BB209_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end37
+; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd17+40];
+; SM60-NEXT: $L__BB209_11: // %atomicrmw.start43
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd31;
+; SM60-NEXT: max.f64 %rd23, %rd14, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
+; SM60-NEXT: @%p6 bra $L__BB209_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end42
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd17+48];
+; SM60-NEXT: $L__BB209_13: // %atomicrmw.start53
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd24, %rd32, %rd11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM60-NEXT: mov.b64 %rd32, %rd15;
+; SM60-NEXT: @%p7 bra $L__BB209_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end52
+; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd17+56];
+; SM60-NEXT: $L__BB209_15: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd25, %rd33, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
+; SM60-NEXT: mov.b64 %rd33, %rd16;
+; SM60-NEXT: @%p8 bra $L__BB209_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end57
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM60-LABEL: fminimum_acq_rel_v2f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [fminimum_acq_rel_v2f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd5];
+; SM60-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM60-NEXT: $L__BB210_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
+; SM60-NEXT: min.f64 %rd6, %rd16, %rd1;
+; SM60-NEXT: selp.f64 %rd7, 0d7FF8000000000000, %rd6, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd16, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd8, %rd16, %rd7, %p2;
+; SM60-NEXT: selp.f64 %rd9, %rd1, %rd8, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd7, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd10, %rd9, %rd7, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
+; SM60-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
+; SM60-NEXT: mov.b64 %rd16, %rd3;
+; SM60-NEXT: @%p5 bra $L__BB210_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd5+8];
+; SM60-NEXT: setp.eq.b64 %p8, %rd2, -9223372036854775808;
+; SM60-NEXT: $L__BB210_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
+; SM60-NEXT: min.f64 %rd11, %rd17, %rd2;
+; SM60-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
+; SM60-NEXT: setp.eq.b64 %p7, %rd17, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd13, %rd17, %rd12, %p7;
+; SM60-NEXT: selp.f64 %rd14, %rd2, %rd13, %p8;
+; SM60-NEXT: setp.eq.f64 %p9, %rd12, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd15, %rd14, %rd12, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
+; SM60-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
+; SM60-NEXT: mov.b64 %rd17, %rd4;
+; SM60-NEXT: @%p10 bra $L__BB210_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM60-LABEL: fminimum_acq_rel_v4f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<21>;
+; SM60-NEXT: .reg .b64 %rd<34>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd9];
+; SM60-NEXT: setp.eq.b64 %p3, %rd3, -9223372036854775808;
+; SM60-NEXT: $L__BB211_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd30;
+; SM60-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
+; SM60-NEXT: min.f64 %rd10, %rd5, %rd3;
+; SM60-NEXT: selp.f64 %rd11, 0d7FF8000000000000, %rd10, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd5, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd12, %rd5, %rd11, %p2;
+; SM60-NEXT: selp.f64 %rd13, %rd3, %rd12, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd11, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
+; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
+; SM60-NEXT: @%p5 bra $L__BB211_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd9+8];
+; SM60-NEXT: setp.eq.b64 %p8, %rd4, -9223372036854775808;
+; SM60-NEXT: $L__BB211_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd31;
+; SM60-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
+; SM60-NEXT: min.f64 %rd15, %rd6, %rd4;
+; SM60-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
+; SM60-NEXT: setp.eq.b64 %p7, %rd6, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd17, %rd6, %rd16, %p7;
+; SM60-NEXT: selp.f64 %rd18, %rd4, %rd17, %p8;
+; SM60-NEXT: setp.eq.f64 %p9, %rd16, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
+; SM60-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
+; SM60-NEXT: @%p10 bra $L__BB211_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd9+16];
+; SM60-NEXT: setp.eq.b64 %p13, %rd1, -9223372036854775808;
+; SM60-NEXT: $L__BB211_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
+; SM60-NEXT: min.f64 %rd20, %rd32, %rd1;
+; SM60-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p11;
+; SM60-NEXT: setp.eq.b64 %p12, %rd32, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd22, %rd32, %rd21, %p12;
+; SM60-NEXT: selp.f64 %rd23, %rd1, %rd22, %p13;
+; SM60-NEXT: setp.eq.f64 %p14, %rd21, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd24, %rd23, %rd21, %p14;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
+; SM60-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
+; SM60-NEXT: mov.b64 %rd32, %rd7;
+; SM60-NEXT: @%p15 bra $L__BB211_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd9+24];
+; SM60-NEXT: setp.eq.b64 %p18, %rd2, -9223372036854775808;
+; SM60-NEXT: $L__BB211_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
+; SM60-NEXT: min.f64 %rd25, %rd33, %rd2;
+; SM60-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p16;
+; SM60-NEXT: setp.eq.b64 %p17, %rd33, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd27, %rd33, %rd26, %p17;
+; SM60-NEXT: selp.f64 %rd28, %rd2, %rd27, %p18;
+; SM60-NEXT: setp.eq.f64 %p19, %rd26, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd29, %rd28, %rd26, %p19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
+; SM60-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
+; SM60-NEXT: mov.b64 %rd33, %rd8;
+; SM60-NEXT: @%p20 bra $L__BB211_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd30, %rd31};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM60-LABEL: fminimum_acq_rel_v8f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<41>;
+; SM60-NEXT: .reg .b64 %rd<66>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fminimum_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd58, [%rd17];
+; SM60-NEXT: setp.eq.b64 %p3, %rd5, -9223372036854775808;
+; SM60-NEXT: $L__BB212_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd58;
+; SM60-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
+; SM60-NEXT: min.f64 %rd18, %rd7, %rd5;
+; SM60-NEXT: selp.f64 %rd19, 0d7FF8000000000000, %rd18, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd7, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd20, %rd7, %rd19, %p2;
+; SM60-NEXT: selp.f64 %rd21, %rd5, %rd20, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd19, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
+; SM60-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
+; SM60-NEXT: @%p5 bra $L__BB212_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd59, [%rd17+8];
+; SM60-NEXT: setp.eq.b64 %p8, %rd6, -9223372036854775808;
+; SM60-NEXT: $L__BB212_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd59;
+; SM60-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
+; SM60-NEXT: min.f64 %rd23, %rd8, %rd6;
+; SM60-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
+; SM60-NEXT: setp.eq.b64 %p7, %rd8, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd25, %rd8, %rd24, %p7;
+; SM60-NEXT: selp.f64 %rd26, %rd6, %rd25, %p8;
+; SM60-NEXT: setp.eq.f64 %p9, %rd24, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
+; SM60-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
+; SM60-NEXT: @%p10 bra $L__BB212_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd60, [%rd17+16];
+; SM60-NEXT: setp.eq.b64 %p13, %rd3, -9223372036854775808;
+; SM60-NEXT: $L__BB212_5: // %atomicrmw.start19
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd60;
+; SM60-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
+; SM60-NEXT: min.f64 %rd28, %rd9, %rd3;
+; SM60-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p11;
+; SM60-NEXT: setp.eq.b64 %p12, %rd9, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd30, %rd9, %rd29, %p12;
+; SM60-NEXT: selp.f64 %rd31, %rd3, %rd30, %p13;
+; SM60-NEXT: setp.eq.f64 %p14, %rd29, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
+; SM60-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
+; SM60-NEXT: @%p15 bra $L__BB212_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end18
+; SM60-NEXT: ld.volatile.global.b64 %rd61, [%rd17+24];
+; SM60-NEXT: setp.eq.b64 %p18, %rd4, -9223372036854775808;
+; SM60-NEXT: $L__BB212_7: // %atomicrmw.start24
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd61;
+; SM60-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
+; SM60-NEXT: min.f64 %rd33, %rd10, %rd4;
+; SM60-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p16;
+; SM60-NEXT: setp.eq.b64 %p17, %rd10, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd35, %rd10, %rd34, %p17;
+; SM60-NEXT: selp.f64 %rd36, %rd4, %rd35, %p18;
+; SM60-NEXT: setp.eq.f64 %p19, %rd34, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
+; SM60-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
+; SM60-NEXT: @%p20 bra $L__BB212_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end23
+; SM60-NEXT: ld.volatile.global.b64 %rd62, [%rd17+32];
+; SM60-NEXT: setp.eq.b64 %p23, %rd1, -9223372036854775808;
+; SM60-NEXT: $L__BB212_9: // %atomicrmw.start38
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd62;
+; SM60-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
+; SM60-NEXT: min.f64 %rd38, %rd13, %rd1;
+; SM60-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p21;
+; SM60-NEXT: setp.eq.b64 %p22, %rd13, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd40, %rd13, %rd39, %p22;
+; SM60-NEXT: selp.f64 %rd41, %rd1, %rd40, %p23;
+; SM60-NEXT: setp.eq.f64 %p24, %rd39, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
+; SM60-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
+; SM60-NEXT: @%p25 bra $L__BB212_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end37
+; SM60-NEXT: ld.volatile.global.b64 %rd63, [%rd17+40];
+; SM60-NEXT: setp.eq.b64 %p28, %rd2, -9223372036854775808;
+; SM60-NEXT: $L__BB212_11: // %atomicrmw.start43
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd63;
+; SM60-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
+; SM60-NEXT: min.f64 %rd43, %rd14, %rd2;
+; SM60-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
+; SM60-NEXT: setp.eq.b64 %p27, %rd14, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd45, %rd14, %rd44, %p27;
+; SM60-NEXT: selp.f64 %rd46, %rd2, %rd45, %p28;
+; SM60-NEXT: setp.eq.f64 %p29, %rd44, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
+; SM60-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
+; SM60-NEXT: @%p30 bra $L__BB212_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end42
+; SM60-NEXT: ld.volatile.global.b64 %rd64, [%rd17+48];
+; SM60-NEXT: setp.eq.b64 %p33, %rd11, -9223372036854775808;
+; SM60-NEXT: $L__BB212_13: // %atomicrmw.start53
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
+; SM60-NEXT: min.f64 %rd48, %rd64, %rd11;
+; SM60-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p31;
+; SM60-NEXT: setp.eq.b64 %p32, %rd64, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd50, %rd64, %rd49, %p32;
+; SM60-NEXT: selp.f64 %rd51, %rd11, %rd50, %p33;
+; SM60-NEXT: setp.eq.f64 %p34, %rd49, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd52, %rd51, %rd49, %p34;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
+; SM60-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
+; SM60-NEXT: mov.b64 %rd64, %rd15;
+; SM60-NEXT: @%p35 bra $L__BB212_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end52
+; SM60-NEXT: ld.volatile.global.b64 %rd65, [%rd17+56];
+; SM60-NEXT: setp.eq.b64 %p38, %rd12, -9223372036854775808;
+; SM60-NEXT: $L__BB212_15: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
+; SM60-NEXT: min.f64 %rd53, %rd65, %rd12;
+; SM60-NEXT: selp.f64 %rd54, 0d7FF8000000000000, %rd53, %p36;
+; SM60-NEXT: setp.eq.b64 %p37, %rd65, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd55, %rd65, %rd54, %p37;
+; SM60-NEXT: selp.f64 %rd56, %rd12, %rd55, %p38;
+; SM60-NEXT: setp.eq.f64 %p39, %rd54, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd57, %rd56, %rd54, %p39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
+; SM60-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
+; SM60-NEXT: mov.b64 %rd65, %rd16;
+; SM60-NEXT: @%p40 bra $L__BB212_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end57
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd62, %rd63};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v2f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [fmaximum_acq_rel_v2f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd5];
+; SM60-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM60-NEXT: $L__BB213_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
+; SM60-NEXT: max.f64 %rd6, %rd16, %rd1;
+; SM60-NEXT: selp.f64 %rd7, 0d7FF8000000000000, %rd6, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd16, 0;
+; SM60-NEXT: selp.f64 %rd8, %rd16, %rd7, %p2;
+; SM60-NEXT: selp.f64 %rd9, %rd1, %rd8, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd7, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd10, %rd9, %rd7, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
+; SM60-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
+; SM60-NEXT: mov.b64 %rd16, %rd3;
+; SM60-NEXT: @%p5 bra $L__BB213_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd5+8];
+; SM60-NEXT: setp.eq.b64 %p8, %rd2, 0;
+; SM60-NEXT: $L__BB213_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
+; SM60-NEXT: max.f64 %rd11, %rd17, %rd2;
+; SM60-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
+; SM60-NEXT: setp.eq.b64 %p7, %rd17, 0;
+; SM60-NEXT: selp.f64 %rd13, %rd17, %rd12, %p7;
+; SM60-NEXT: selp.f64 %rd14, %rd2, %rd13, %p8;
+; SM60-NEXT: setp.eq.f64 %p9, %rd12, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd15, %rd14, %rd12, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
+; SM60-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
+; SM60-NEXT: mov.b64 %rd17, %rd4;
+; SM60-NEXT: @%p10 bra $L__BB213_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v4f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<21>;
+; SM60-NEXT: .reg .b64 %rd<34>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd9];
+; SM60-NEXT: setp.eq.b64 %p3, %rd3, 0;
+; SM60-NEXT: $L__BB214_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd30;
+; SM60-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
+; SM60-NEXT: max.f64 %rd10, %rd5, %rd3;
+; SM60-NEXT: selp.f64 %rd11, 0d7FF8000000000000, %rd10, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd5, 0;
+; SM60-NEXT: selp.f64 %rd12, %rd5, %rd11, %p2;
+; SM60-NEXT: selp.f64 %rd13, %rd3, %rd12, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd11, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
+; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
+; SM60-NEXT: @%p5 bra $L__BB214_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd9+8];
+; SM60-NEXT: setp.eq.b64 %p8, %rd4, 0;
+; SM60-NEXT: $L__BB214_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd31;
+; SM60-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
+; SM60-NEXT: max.f64 %rd15, %rd6, %rd4;
+; SM60-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
+; SM60-NEXT: setp.eq.b64 %p7, %rd6, 0;
+; SM60-NEXT: selp.f64 %rd17, %rd6, %rd16, %p7;
+; SM60-NEXT: selp.f64 %rd18, %rd4, %rd17, %p8;
+; SM60-NEXT: setp.eq.f64 %p9, %rd16, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
+; SM60-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
+; SM60-NEXT: @%p10 bra $L__BB214_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd9+16];
+; SM60-NEXT: setp.eq.b64 %p13, %rd1, 0;
+; SM60-NEXT: $L__BB214_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
+; SM60-NEXT: max.f64 %rd20, %rd32, %rd1;
+; SM60-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p11;
+; SM60-NEXT: setp.eq.b64 %p12, %rd32, 0;
+; SM60-NEXT: selp.f64 %rd22, %rd32, %rd21, %p12;
+; SM60-NEXT: selp.f64 %rd23, %rd1, %rd22, %p13;
+; SM60-NEXT: setp.eq.f64 %p14, %rd21, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd24, %rd23, %rd21, %p14;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
+; SM60-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
+; SM60-NEXT: mov.b64 %rd32, %rd7;
+; SM60-NEXT: @%p15 bra $L__BB214_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd9+24];
+; SM60-NEXT: setp.eq.b64 %p18, %rd2, 0;
+; SM60-NEXT: $L__BB214_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
+; SM60-NEXT: max.f64 %rd25, %rd33, %rd2;
+; SM60-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p16;
+; SM60-NEXT: setp.eq.b64 %p17, %rd33, 0;
+; SM60-NEXT: selp.f64 %rd27, %rd33, %rd26, %p17;
+; SM60-NEXT: selp.f64 %rd28, %rd2, %rd27, %p18;
+; SM60-NEXT: setp.eq.f64 %p19, %rd26, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd29, %rd28, %rd26, %p19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
+; SM60-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
+; SM60-NEXT: mov.b64 %rd33, %rd8;
+; SM60-NEXT: @%p20 bra $L__BB214_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd30, %rd31};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v8f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<41>;
+; SM60-NEXT: .reg .b64 %rd<66>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmaximum_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd58, [%rd17];
+; SM60-NEXT: setp.eq.b64 %p3, %rd5, 0;
+; SM60-NEXT: $L__BB215_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd58;
+; SM60-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
+; SM60-NEXT: max.f64 %rd18, %rd7, %rd5;
+; SM60-NEXT: selp.f64 %rd19, 0d7FF8000000000000, %rd18, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd7, 0;
+; SM60-NEXT: selp.f64 %rd20, %rd7, %rd19, %p2;
+; SM60-NEXT: selp.f64 %rd21, %rd5, %rd20, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd19, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
+; SM60-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
+; SM60-NEXT: @%p5 bra $L__BB215_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd59, [%rd17+8];
+; SM60-NEXT: setp.eq.b64 %p8, %rd6, 0;
+; SM60-NEXT: $L__BB215_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd59;
+; SM60-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
+; SM60-NEXT: max.f64 %rd23, %rd8, %rd6;
+; SM60-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
+; SM60-NEXT: setp.eq.b64 %p7, %rd8, 0;
+; SM60-NEXT: selp.f64 %rd25, %rd8, %rd24, %p7;
+; SM60-NEXT: selp.f64 %rd26, %rd6, %rd25, %p8;
+; SM60-NEXT: setp.eq.f64 %p9, %rd24, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
+; SM60-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
+; SM60-NEXT: @%p10 bra $L__BB215_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd60, [%rd17+16];
+; SM60-NEXT: setp.eq.b64 %p13, %rd3, 0;
+; SM60-NEXT: $L__BB215_5: // %atomicrmw.start19
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd60;
+; SM60-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
+; SM60-NEXT: max.f64 %rd28, %rd9, %rd3;
+; SM60-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p11;
+; SM60-NEXT: setp.eq.b64 %p12, %rd9, 0;
+; SM60-NEXT: selp.f64 %rd30, %rd9, %rd29, %p12;
+; SM60-NEXT: selp.f64 %rd31, %rd3, %rd30, %p13;
+; SM60-NEXT: setp.eq.f64 %p14, %rd29, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
+; SM60-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
+; SM60-NEXT: @%p15 bra $L__BB215_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end18
+; SM60-NEXT: ld.volatile.global.b64 %rd61, [%rd17+24];
+; SM60-NEXT: setp.eq.b64 %p18, %rd4, 0;
+; SM60-NEXT: $L__BB215_7: // %atomicrmw.start24
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd61;
+; SM60-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
+; SM60-NEXT: max.f64 %rd33, %rd10, %rd4;
+; SM60-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p16;
+; SM60-NEXT: setp.eq.b64 %p17, %rd10, 0;
+; SM60-NEXT: selp.f64 %rd35, %rd10, %rd34, %p17;
+; SM60-NEXT: selp.f64 %rd36, %rd4, %rd35, %p18;
+; SM60-NEXT: setp.eq.f64 %p19, %rd34, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
+; SM60-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
+; SM60-NEXT: @%p20 bra $L__BB215_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end23
+; SM60-NEXT: ld.volatile.global.b64 %rd62, [%rd17+32];
+; SM60-NEXT: setp.eq.b64 %p23, %rd1, 0;
+; SM60-NEXT: $L__BB215_9: // %atomicrmw.start38
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd62;
+; SM60-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
+; SM60-NEXT: max.f64 %rd38, %rd13, %rd1;
+; SM60-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p21;
+; SM60-NEXT: setp.eq.b64 %p22, %rd13, 0;
+; SM60-NEXT: selp.f64 %rd40, %rd13, %rd39, %p22;
+; SM60-NEXT: selp.f64 %rd41, %rd1, %rd40, %p23;
+; SM60-NEXT: setp.eq.f64 %p24, %rd39, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
+; SM60-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
+; SM60-NEXT: @%p25 bra $L__BB215_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end37
+; SM60-NEXT: ld.volatile.global.b64 %rd63, [%rd17+40];
+; SM60-NEXT: setp.eq.b64 %p28, %rd2, 0;
+; SM60-NEXT: $L__BB215_11: // %atomicrmw.start43
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd63;
+; SM60-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
+; SM60-NEXT: max.f64 %rd43, %rd14, %rd2;
+; SM60-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
+; SM60-NEXT: setp.eq.b64 %p27, %rd14, 0;
+; SM60-NEXT: selp.f64 %rd45, %rd14, %rd44, %p27;
+; SM60-NEXT: selp.f64 %rd46, %rd2, %rd45, %p28;
+; SM60-NEXT: setp.eq.f64 %p29, %rd44, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
+; SM60-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
+; SM60-NEXT: @%p30 bra $L__BB215_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end42
+; SM60-NEXT: ld.volatile.global.b64 %rd64, [%rd17+48];
+; SM60-NEXT: setp.eq.b64 %p33, %rd11, 0;
+; SM60-NEXT: $L__BB215_13: // %atomicrmw.start53
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
+; SM60-NEXT: max.f64 %rd48, %rd64, %rd11;
+; SM60-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p31;
+; SM60-NEXT: setp.eq.b64 %p32, %rd64, 0;
+; SM60-NEXT: selp.f64 %rd50, %rd64, %rd49, %p32;
+; SM60-NEXT: selp.f64 %rd51, %rd11, %rd50, %p33;
+; SM60-NEXT: setp.eq.f64 %p34, %rd49, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd52, %rd51, %rd49, %p34;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
+; SM60-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
+; SM60-NEXT: mov.b64 %rd64, %rd15;
+; SM60-NEXT: @%p35 bra $L__BB215_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end52
+; SM60-NEXT: ld.volatile.global.b64 %rd65, [%rd17+56];
+; SM60-NEXT: setp.eq.b64 %p38, %rd12, 0;
+; SM60-NEXT: $L__BB215_15: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
+; SM60-NEXT: max.f64 %rd53, %rd65, %rd12;
+; SM60-NEXT: selp.f64 %rd54, 0d7FF8000000000000, %rd53, %p36;
+; SM60-NEXT: setp.eq.b64 %p37, %rd65, 0;
+; SM60-NEXT: selp.f64 %rd55, %rd65, %rd54, %p37;
+; SM60-NEXT: selp.f64 %rd56, %rd12, %rd55, %p38;
+; SM60-NEXT: setp.eq.f64 %p39, %rd54, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd57, %rd56, %rd54, %p39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
+; SM60-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
+; SM60-NEXT: mov.b64 %rd65, %rd16;
+; SM60-NEXT: @%p40 bra $L__BB215_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end57
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd62, %rd63};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM60-LABEL: fadd_acq_rel_v2f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fadd_acq_rel_v2f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB216_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.rn.f16x2 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB216_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM60-LABEL: fadd_acq_rel_v4f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB217_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.rn.f16x2 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: add.rn.f16x2 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB217_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM60-LABEL: fadd_acq_rel_v8f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB218_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.rn.f16x2 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: add.rn.f16x2 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB218_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB218_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.rn.f16x2 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: add.rn.f16x2 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB218_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM60-LABEL: fsub_acq_rel_v2f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB219_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f16x2 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB219_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM60-LABEL: fsub_acq_rel_v4f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB220_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f16x2 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: sub.rn.f16x2 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB220_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM60-LABEL: fsub_acq_rel_v8f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB221_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f16x2 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: sub.rn.f16x2 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB221_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB221_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f16x2 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: sub.rn.f16x2 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB221_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM60-LABEL: fmin_acq_rel_v2f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<11>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r10, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM60-NEXT: $L__BB222_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r10;
+; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM60-NEXT: min.f32 %r5, %r4, %r3;
+; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM60-NEXT: min.f32 %r8, %r7, %r6;
+; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM60-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r10;
+; SM60-NEXT: mov.b32 %r10, %r1;
+; SM60-NEXT: @%p1 bra $L__BB222_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM60-LABEL: fmin_acq_rel_v4f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: cvt.f32.f16 %r10, %rs8;
+; SM60-NEXT: cvt.f32.f16 %r13, %rs7;
+; SM60-NEXT: $L__BB223_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r17;
+; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM60-NEXT: min.f32 %r5, %r4, %r3;
+; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM60-NEXT: min.f32 %r8, %r7, %r6;
+; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM60-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r18;
+; SM60-NEXT: cvt.f32.f16 %r11, %rs10;
+; SM60-NEXT: min.f32 %r12, %r11, %r10;
+; SM60-NEXT: cvt.rn.f16.f32 %rs11, %r12;
+; SM60-NEXT: cvt.f32.f16 %r14, %rs9;
+; SM60-NEXT: min.f32 %r15, %r14, %r13;
+; SM60-NEXT: cvt.rn.f16.f32 %rs12, %r15;
+; SM60-NEXT: mov.b32 %r16, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r16;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB223_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r17, %r18};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM60-LABEL: fmin_acq_rel_v8f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<37>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.f32.f16 %r5, %rs2;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: cvt.f32.f16 %r12, %rs8;
+; SM60-NEXT: $L__BB224_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r33;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs4;
+; SM60-NEXT: min.f32 %r7, %r6, %r5;
+; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r7;
+; SM60-NEXT: cvt.f32.f16 %r8, %rs1;
+; SM60-NEXT: cvt.f32.f16 %r9, %rs3;
+; SM60-NEXT: min.f32 %r10, %r9, %r8;
+; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r10;
+; SM60-NEXT: mov.b32 %r11, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r34;
+; SM60-NEXT: cvt.f32.f16 %r13, %rs10;
+; SM60-NEXT: min.f32 %r14, %r13, %r12;
+; SM60-NEXT: cvt.rn.f16.f32 %rs11, %r14;
+; SM60-NEXT: cvt.f32.f16 %r15, %rs7;
+; SM60-NEXT: cvt.f32.f16 %r16, %rs9;
+; SM60-NEXT: min.f32 %r17, %r16, %r15;
+; SM60-NEXT: cvt.rn.f16.f32 %rs12, %r17;
+; SM60-NEXT: mov.b32 %r18, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB224_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r35, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: cvt.f32.f16 %r19, %rs14;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: cvt.f32.f16 %r26, %rs20;
+; SM60-NEXT: $L__BB224_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r35;
+; SM60-NEXT: cvt.f32.f16 %r20, %rs16;
+; SM60-NEXT: min.f32 %r21, %r20, %r19;
+; SM60-NEXT: cvt.rn.f16.f32 %rs17, %r21;
+; SM60-NEXT: cvt.f32.f16 %r22, %rs13;
+; SM60-NEXT: cvt.f32.f16 %r23, %rs15;
+; SM60-NEXT: min.f32 %r24, %r23, %r22;
+; SM60-NEXT: cvt.rn.f16.f32 %rs18, %r24;
+; SM60-NEXT: mov.b32 %r25, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r36;
+; SM60-NEXT: cvt.f32.f16 %r27, %rs22;
+; SM60-NEXT: min.f32 %r28, %r27, %r26;
+; SM60-NEXT: cvt.rn.f16.f32 %rs23, %r28;
+; SM60-NEXT: cvt.f32.f16 %r29, %rs19;
+; SM60-NEXT: cvt.f32.f16 %r30, %rs21;
+; SM60-NEXT: min.f32 %r31, %r30, %r29;
+; SM60-NEXT: cvt.rn.f16.f32 %rs24, %r31;
+; SM60-NEXT: mov.b32 %r32, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r32;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r35;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r36;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r35, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB224_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM60-LABEL: fmax_acq_rel_v2f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<11>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r10, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM60-NEXT: $L__BB225_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r10;
+; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM60-NEXT: max.f32 %r5, %r4, %r3;
+; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM60-NEXT: max.f32 %r8, %r7, %r6;
+; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM60-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r10;
+; SM60-NEXT: mov.b32 %r10, %r1;
+; SM60-NEXT: @%p1 bra $L__BB225_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM60-LABEL: fmax_acq_rel_v4f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: cvt.f32.f16 %r10, %rs8;
+; SM60-NEXT: cvt.f32.f16 %r13, %rs7;
+; SM60-NEXT: $L__BB226_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r17;
+; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM60-NEXT: max.f32 %r5, %r4, %r3;
+; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM60-NEXT: max.f32 %r8, %r7, %r6;
+; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM60-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r18;
+; SM60-NEXT: cvt.f32.f16 %r11, %rs10;
+; SM60-NEXT: max.f32 %r12, %r11, %r10;
+; SM60-NEXT: cvt.rn.f16.f32 %rs11, %r12;
+; SM60-NEXT: cvt.f32.f16 %r14, %rs9;
+; SM60-NEXT: max.f32 %r15, %r14, %r13;
+; SM60-NEXT: cvt.rn.f16.f32 %rs12, %r15;
+; SM60-NEXT: mov.b32 %r16, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r16;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB226_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r17, %r18};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM60-LABEL: fmax_acq_rel_v8f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<37>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.f32.f16 %r5, %rs2;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: cvt.f32.f16 %r12, %rs8;
+; SM60-NEXT: $L__BB227_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r33;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs4;
+; SM60-NEXT: max.f32 %r7, %r6, %r5;
+; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r7;
+; SM60-NEXT: cvt.f32.f16 %r8, %rs1;
+; SM60-NEXT: cvt.f32.f16 %r9, %rs3;
+; SM60-NEXT: max.f32 %r10, %r9, %r8;
+; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r10;
+; SM60-NEXT: mov.b32 %r11, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r34;
+; SM60-NEXT: cvt.f32.f16 %r13, %rs10;
+; SM60-NEXT: max.f32 %r14, %r13, %r12;
+; SM60-NEXT: cvt.rn.f16.f32 %rs11, %r14;
+; SM60-NEXT: cvt.f32.f16 %r15, %rs7;
+; SM60-NEXT: cvt.f32.f16 %r16, %rs9;
+; SM60-NEXT: max.f32 %r17, %r16, %r15;
+; SM60-NEXT: cvt.rn.f16.f32 %rs12, %r17;
+; SM60-NEXT: mov.b32 %r18, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB227_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r35, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: cvt.f32.f16 %r19, %rs14;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: cvt.f32.f16 %r26, %rs20;
+; SM60-NEXT: $L__BB227_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r35;
+; SM60-NEXT: cvt.f32.f16 %r20, %rs16;
+; SM60-NEXT: max.f32 %r21, %r20, %r19;
+; SM60-NEXT: cvt.rn.f16.f32 %rs17, %r21;
+; SM60-NEXT: cvt.f32.f16 %r22, %rs13;
+; SM60-NEXT: cvt.f32.f16 %r23, %rs15;
+; SM60-NEXT: max.f32 %r24, %r23, %r22;
+; SM60-NEXT: cvt.rn.f16.f32 %rs18, %r24;
+; SM60-NEXT: mov.b32 %r25, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r36;
+; SM60-NEXT: cvt.f32.f16 %r27, %rs22;
+; SM60-NEXT: max.f32 %r28, %r27, %r26;
+; SM60-NEXT: cvt.rn.f16.f32 %rs23, %r28;
+; SM60-NEXT: cvt.f32.f16 %r29, %rs19;
+; SM60-NEXT: cvt.f32.f16 %r30, %rs21;
+; SM60-NEXT: max.f32 %r31, %r30, %r29;
+; SM60-NEXT: cvt.rn.f16.f32 %rs24, %r31;
+; SM60-NEXT: mov.b32 %r32, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r32;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r35;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r36;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r35, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB227_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM60-LABEL: fminimum_acq_rel_v2f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<12>;
+; SM60-NEXT: .reg .b16 %rs<16>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM60-NEXT: $L__BB228_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: mov.b16 %rs9, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM60-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p11, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p11 bra $L__BB228_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM60-LABEL: fminimum_acq_rel_v4f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<22>;
+; SM60-NEXT: .reg .b16 %rs<30>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM60-NEXT: setp.eq.b16 %p14, %rs17, -32768;
+; SM60-NEXT: setp.eq.b16 %p19, %rs16, -32768;
+; SM60-NEXT: $L__BB229_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: mov.b16 %rs9, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM60-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: mov.b32 {%rs18, %rs19}, %r6;
+; SM60-NEXT: setp.lt.f16 %p11, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM60-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs19, -32768;
+; SM60-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM60-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM60-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM60-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM60-NEXT: setp.lt.f16 %p16, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM60-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs18, -32768;
+; SM60-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM60-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM60-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM60-NEXT: mov.b32 %r4, {%rs29, %rs24};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB229_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM60-LABEL: fminimum_acq_rel_v8f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<43>;
+; SM60-NEXT: .reg .b16 %rs<59>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM60-NEXT: setp.eq.b16 %p14, %rs17, -32768;
+; SM60-NEXT: setp.eq.b16 %p19, %rs16, -32768;
+; SM60-NEXT: $L__BB230_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: mov.b16 %rs9, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM60-NEXT: mov.b32 %r5, {%rs15, %rs10};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: mov.b32 {%rs18, %rs19}, %r10;
+; SM60-NEXT: setp.lt.f16 %p11, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM60-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs19, -32768;
+; SM60-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM60-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM60-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM60-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM60-NEXT: setp.lt.f16 %p16, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM60-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs18, -32768;
+; SM60-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM60-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM60-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM60-NEXT: mov.b32 %r6, {%rs29, %rs24};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB230_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB230_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs30, %rs31}, %r3;
+; SM60-NEXT: mov.b32 {%rs32, %rs33}, %r11;
+; SM60-NEXT: setp.lt.f16 %p22, %rs33, %rs31;
+; SM60-NEXT: selp.b16 %rs34, %rs33, %rs31, %p22;
+; SM60-NEXT: setp.nan.f16 %p23, %rs33, %rs31;
+; SM60-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p23;
+; SM60-NEXT: setp.eq.b16 %p24, %rs33, -32768;
+; SM60-NEXT: selp.b16 %rs36, %rs33, %rs35, %p24;
+; SM60-NEXT: setp.eq.b16 %p25, %rs31, -32768;
+; SM60-NEXT: selp.b16 %rs37, %rs31, %rs36, %p25;
+; SM60-NEXT: mov.b16 %rs38, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p26, %rs35, %rs38;
+; SM60-NEXT: selp.b16 %rs39, %rs37, %rs35, %p26;
+; SM60-NEXT: setp.lt.f16 %p27, %rs32, %rs30;
+; SM60-NEXT: selp.b16 %rs40, %rs32, %rs30, %p27;
+; SM60-NEXT: setp.nan.f16 %p28, %rs32, %rs30;
+; SM60-NEXT: selp.b16 %rs41, 0x7E00, %rs40, %p28;
+; SM60-NEXT: setp.eq.b16 %p29, %rs32, -32768;
+; SM60-NEXT: selp.b16 %rs42, %rs32, %rs41, %p29;
+; SM60-NEXT: setp.eq.b16 %p30, %rs30, -32768;
+; SM60-NEXT: selp.b16 %rs43, %rs30, %rs42, %p30;
+; SM60-NEXT: setp.eq.f16 %p31, %rs41, %rs38;
+; SM60-NEXT: selp.b16 %rs44, %rs43, %rs41, %p31;
+; SM60-NEXT: mov.b32 %r7, {%rs44, %rs39};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: mov.b32 {%rs45, %rs46}, %r4;
+; SM60-NEXT: mov.b32 {%rs47, %rs48}, %r12;
+; SM60-NEXT: setp.lt.f16 %p32, %rs48, %rs46;
+; SM60-NEXT: selp.b16 %rs49, %rs48, %rs46, %p32;
+; SM60-NEXT: setp.nan.f16 %p33, %rs48, %rs46;
+; SM60-NEXT: selp.b16 %rs50, 0x7E00, %rs49, %p33;
+; SM60-NEXT: setp.eq.b16 %p34, %rs48, -32768;
+; SM60-NEXT: selp.b16 %rs51, %rs48, %rs50, %p34;
+; SM60-NEXT: setp.eq.b16 %p35, %rs46, -32768;
+; SM60-NEXT: selp.b16 %rs52, %rs46, %rs51, %p35;
+; SM60-NEXT: setp.eq.f16 %p36, %rs50, %rs38;
+; SM60-NEXT: selp.b16 %rs53, %rs52, %rs50, %p36;
+; SM60-NEXT: setp.lt.f16 %p37, %rs47, %rs45;
+; SM60-NEXT: selp.b16 %rs54, %rs47, %rs45, %p37;
+; SM60-NEXT: setp.nan.f16 %p38, %rs47, %rs45;
+; SM60-NEXT: selp.b16 %rs55, 0x7E00, %rs54, %p38;
+; SM60-NEXT: setp.eq.b16 %p39, %rs47, -32768;
+; SM60-NEXT: selp.b16 %rs56, %rs47, %rs55, %p39;
+; SM60-NEXT: setp.eq.b16 %p40, %rs45, -32768;
+; SM60-NEXT: selp.b16 %rs57, %rs45, %rs56, %p40;
+; SM60-NEXT: setp.eq.f16 %p41, %rs55, %rs38;
+; SM60-NEXT: selp.b16 %rs58, %rs57, %rs55, %p41;
+; SM60-NEXT: mov.b32 %r8, {%rs58, %rs53};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p42 bra $L__BB230_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v2f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<12>;
+; SM60-NEXT: .reg .b16 %rs<16>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM60-NEXT: $L__BB231_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: mov.b16 %rs9, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM60-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p11, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p11 bra $L__BB231_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v4f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<22>;
+; SM60-NEXT: .reg .b16 %rs<30>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM60-NEXT: setp.eq.b16 %p14, %rs17, 0;
+; SM60-NEXT: setp.eq.b16 %p19, %rs16, 0;
+; SM60-NEXT: $L__BB232_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: mov.b16 %rs9, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM60-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: mov.b32 {%rs18, %rs19}, %r6;
+; SM60-NEXT: setp.gt.f16 %p11, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM60-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs19, 0;
+; SM60-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM60-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM60-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM60-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM60-NEXT: setp.gt.f16 %p16, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM60-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs18, 0;
+; SM60-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM60-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM60-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM60-NEXT: mov.b32 %r4, {%rs29, %rs24};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB232_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v8f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<43>;
+; SM60-NEXT: .reg .b16 %rs<59>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM60-NEXT: setp.eq.b16 %p14, %rs17, 0;
+; SM60-NEXT: setp.eq.b16 %p19, %rs16, 0;
+; SM60-NEXT: $L__BB233_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: mov.b16 %rs9, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM60-NEXT: mov.b32 %r5, {%rs15, %rs10};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: mov.b32 {%rs18, %rs19}, %r10;
+; SM60-NEXT: setp.gt.f16 %p11, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM60-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs19, 0;
+; SM60-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM60-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM60-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM60-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM60-NEXT: setp.gt.f16 %p16, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM60-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs18, 0;
+; SM60-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM60-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM60-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM60-NEXT: mov.b32 %r6, {%rs29, %rs24};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB233_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB233_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs30, %rs31}, %r3;
+; SM60-NEXT: mov.b32 {%rs32, %rs33}, %r11;
+; SM60-NEXT: setp.gt.f16 %p22, %rs33, %rs31;
+; SM60-NEXT: selp.b16 %rs34, %rs33, %rs31, %p22;
+; SM60-NEXT: setp.nan.f16 %p23, %rs33, %rs31;
+; SM60-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p23;
+; SM60-NEXT: setp.eq.b16 %p24, %rs33, 0;
+; SM60-NEXT: selp.b16 %rs36, %rs33, %rs35, %p24;
+; SM60-NEXT: setp.eq.b16 %p25, %rs31, 0;
+; SM60-NEXT: selp.b16 %rs37, %rs31, %rs36, %p25;
+; SM60-NEXT: mov.b16 %rs38, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p26, %rs35, %rs38;
+; SM60-NEXT: selp.b16 %rs39, %rs37, %rs35, %p26;
+; SM60-NEXT: setp.gt.f16 %p27, %rs32, %rs30;
+; SM60-NEXT: selp.b16 %rs40, %rs32, %rs30, %p27;
+; SM60-NEXT: setp.nan.f16 %p28, %rs32, %rs30;
+; SM60-NEXT: selp.b16 %rs41, 0x7E00, %rs40, %p28;
+; SM60-NEXT: setp.eq.b16 %p29, %rs32, 0;
+; SM60-NEXT: selp.b16 %rs42, %rs32, %rs41, %p29;
+; SM60-NEXT: setp.eq.b16 %p30, %rs30, 0;
+; SM60-NEXT: selp.b16 %rs43, %rs30, %rs42, %p30;
+; SM60-NEXT: setp.eq.f16 %p31, %rs41, %rs38;
+; SM60-NEXT: selp.b16 %rs44, %rs43, %rs41, %p31;
+; SM60-NEXT: mov.b32 %r7, {%rs44, %rs39};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: mov.b32 {%rs45, %rs46}, %r4;
+; SM60-NEXT: mov.b32 {%rs47, %rs48}, %r12;
+; SM60-NEXT: setp.gt.f16 %p32, %rs48, %rs46;
+; SM60-NEXT: selp.b16 %rs49, %rs48, %rs46, %p32;
+; SM60-NEXT: setp.nan.f16 %p33, %rs48, %rs46;
+; SM60-NEXT: selp.b16 %rs50, 0x7E00, %rs49, %p33;
+; SM60-NEXT: setp.eq.b16 %p34, %rs48, 0;
+; SM60-NEXT: selp.b16 %rs51, %rs48, %rs50, %p34;
+; SM60-NEXT: setp.eq.b16 %p35, %rs46, 0;
+; SM60-NEXT: selp.b16 %rs52, %rs46, %rs51, %p35;
+; SM60-NEXT: setp.eq.f16 %p36, %rs50, %rs38;
+; SM60-NEXT: selp.b16 %rs53, %rs52, %rs50, %p36;
+; SM60-NEXT: setp.gt.f16 %p37, %rs47, %rs45;
+; SM60-NEXT: selp.b16 %rs54, %rs47, %rs45, %p37;
+; SM60-NEXT: setp.nan.f16 %p38, %rs47, %rs45;
+; SM60-NEXT: selp.b16 %rs55, 0x7E00, %rs54, %p38;
+; SM60-NEXT: setp.eq.b16 %p39, %rs47, 0;
+; SM60-NEXT: selp.b16 %rs56, %rs47, %rs55, %p39;
+; SM60-NEXT: setp.eq.b16 %p40, %rs45, 0;
+; SM60-NEXT: selp.b16 %rs57, %rs45, %rs56, %p40;
+; SM60-NEXT: setp.eq.f16 %p41, %rs55, %rs38;
+; SM60-NEXT: selp.b16 %rs58, %rs57, %rs55, %p41;
+; SM60-NEXT: mov.b32 %r8, {%rs58, %rs53};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p42 bra $L__BB233_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM60-LABEL: fadd_acq_rel_v2bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fadd_acq_rel_v2bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r24, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: $L__BB234_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: add.rn.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: add.rn.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM60-NEXT: mov.b32 %r24, %r1;
+; SM60-NEXT: @%p3 bra $L__BB234_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM60-LABEL: fadd_acq_rel_v4bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<47>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
+; SM60-NEXT: $L__BB235_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: add.rn.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: add.rn.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: add.rn.f32 %r28, %r27, %r25;
+; SM60-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM60-NEXT: add.s32 %r30, %r29, %r28;
+; SM60-NEXT: add.s32 %r31, %r30, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM60-NEXT: or.b32 %r32, %r28, 4194304;
+; SM60-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: add.rn.f32 %r38, %r37, %r35;
+; SM60-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM60-NEXT: add.s32 %r40, %r39, %r38;
+; SM60-NEXT: add.s32 %r41, %r40, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM60-NEXT: or.b32 %r42, %r38, 4194304;
+; SM60-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM60-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB235_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM60-LABEL: fadd_acq_rel_v8bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<93>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: $L__BB236_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: add.rn.f32 %r9, %r8, %r6;
+; SM60-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM60-NEXT: add.s32 %r11, %r10, %r9;
+; SM60-NEXT: add.s32 %r12, %r11, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM60-NEXT: or.b32 %r13, %r9, 4194304;
+; SM60-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM60-NEXT: shl.b32 %r18, %r17, 16;
+; SM60-NEXT: add.rn.f32 %r19, %r18, %r16;
+; SM60-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM60-NEXT: add.s32 %r21, %r20, %r19;
+; SM60-NEXT: add.s32 %r22, %r21, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM60-NEXT: or.b32 %r23, %r19, 4194304;
+; SM60-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM60-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM60-NEXT: shl.b32 %r29, %r28, 16;
+; SM60-NEXT: add.rn.f32 %r30, %r29, %r27;
+; SM60-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM60-NEXT: add.s32 %r32, %r31, %r30;
+; SM60-NEXT: add.s32 %r33, %r32, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM60-NEXT: or.b32 %r34, %r30, 4194304;
+; SM60-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM60-NEXT: shl.b32 %r39, %r38, 16;
+; SM60-NEXT: add.rn.f32 %r40, %r39, %r37;
+; SM60-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM60-NEXT: add.s32 %r42, %r41, %r40;
+; SM60-NEXT: add.s32 %r43, %r42, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM60-NEXT: or.b32 %r44, %r40, 4194304;
+; SM60-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM60-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB236_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM60-NEXT: $L__BB236_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM60-NEXT: shl.b32 %r48, %r47, 16;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM60-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM60-NEXT: shl.b32 %r50, %r49, 16;
+; SM60-NEXT: add.rn.f32 %r51, %r50, %r48;
+; SM60-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM60-NEXT: add.s32 %r53, %r52, %r51;
+; SM60-NEXT: add.s32 %r54, %r53, 32767;
+; SM60-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM60-NEXT: or.b32 %r55, %r51, 4194304;
+; SM60-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM60-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM60-NEXT: shl.b32 %r58, %r57, 16;
+; SM60-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM60-NEXT: shl.b32 %r60, %r59, 16;
+; SM60-NEXT: add.rn.f32 %r61, %r60, %r58;
+; SM60-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM60-NEXT: add.s32 %r63, %r62, %r61;
+; SM60-NEXT: add.s32 %r64, %r63, 32767;
+; SM60-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM60-NEXT: or.b32 %r65, %r61, 4194304;
+; SM60-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM60-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM60-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM60-NEXT: shl.b32 %r69, %r68, 16;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM60-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM60-NEXT: shl.b32 %r71, %r70, 16;
+; SM60-NEXT: add.rn.f32 %r72, %r71, %r69;
+; SM60-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM60-NEXT: add.s32 %r74, %r73, %r72;
+; SM60-NEXT: add.s32 %r75, %r74, 32767;
+; SM60-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM60-NEXT: or.b32 %r76, %r72, 4194304;
+; SM60-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM60-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM60-NEXT: shl.b32 %r79, %r78, 16;
+; SM60-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM60-NEXT: shl.b32 %r81, %r80, 16;
+; SM60-NEXT: add.rn.f32 %r82, %r81, %r79;
+; SM60-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM60-NEXT: add.s32 %r84, %r83, %r82;
+; SM60-NEXT: add.s32 %r85, %r84, 32767;
+; SM60-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM60-NEXT: or.b32 %r86, %r82, 4194304;
+; SM60-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM60-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB236_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM60-LABEL: fsub_acq_rel_v2bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r24, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: $L__BB237_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: sub.rn.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: sub.rn.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM60-NEXT: mov.b32 %r24, %r1;
+; SM60-NEXT: @%p3 bra $L__BB237_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM60-LABEL: fsub_acq_rel_v4bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<47>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
+; SM60-NEXT: $L__BB238_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: sub.rn.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: sub.rn.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: sub.rn.f32 %r28, %r27, %r25;
+; SM60-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM60-NEXT: add.s32 %r30, %r29, %r28;
+; SM60-NEXT: add.s32 %r31, %r30, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM60-NEXT: or.b32 %r32, %r28, 4194304;
+; SM60-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: sub.rn.f32 %r38, %r37, %r35;
+; SM60-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM60-NEXT: add.s32 %r40, %r39, %r38;
+; SM60-NEXT: add.s32 %r41, %r40, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM60-NEXT: or.b32 %r42, %r38, 4194304;
+; SM60-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM60-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB238_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM60-LABEL: fsub_acq_rel_v8bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<93>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: $L__BB239_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: sub.rn.f32 %r9, %r8, %r6;
+; SM60-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM60-NEXT: add.s32 %r11, %r10, %r9;
+; SM60-NEXT: add.s32 %r12, %r11, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM60-NEXT: or.b32 %r13, %r9, 4194304;
+; SM60-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM60-NEXT: shl.b32 %r18, %r17, 16;
+; SM60-NEXT: sub.rn.f32 %r19, %r18, %r16;
+; SM60-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM60-NEXT: add.s32 %r21, %r20, %r19;
+; SM60-NEXT: add.s32 %r22, %r21, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM60-NEXT: or.b32 %r23, %r19, 4194304;
+; SM60-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM60-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM60-NEXT: shl.b32 %r29, %r28, 16;
+; SM60-NEXT: sub.rn.f32 %r30, %r29, %r27;
+; SM60-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM60-NEXT: add.s32 %r32, %r31, %r30;
+; SM60-NEXT: add.s32 %r33, %r32, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM60-NEXT: or.b32 %r34, %r30, 4194304;
+; SM60-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM60-NEXT: shl.b32 %r39, %r38, 16;
+; SM60-NEXT: sub.rn.f32 %r40, %r39, %r37;
+; SM60-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM60-NEXT: add.s32 %r42, %r41, %r40;
+; SM60-NEXT: add.s32 %r43, %r42, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM60-NEXT: or.b32 %r44, %r40, 4194304;
+; SM60-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM60-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB239_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM60-NEXT: $L__BB239_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM60-NEXT: shl.b32 %r48, %r47, 16;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM60-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM60-NEXT: shl.b32 %r50, %r49, 16;
+; SM60-NEXT: sub.rn.f32 %r51, %r50, %r48;
+; SM60-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM60-NEXT: add.s32 %r53, %r52, %r51;
+; SM60-NEXT: add.s32 %r54, %r53, 32767;
+; SM60-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM60-NEXT: or.b32 %r55, %r51, 4194304;
+; SM60-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM60-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM60-NEXT: shl.b32 %r58, %r57, 16;
+; SM60-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM60-NEXT: shl.b32 %r60, %r59, 16;
+; SM60-NEXT: sub.rn.f32 %r61, %r60, %r58;
+; SM60-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM60-NEXT: add.s32 %r63, %r62, %r61;
+; SM60-NEXT: add.s32 %r64, %r63, 32767;
+; SM60-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM60-NEXT: or.b32 %r65, %r61, 4194304;
+; SM60-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM60-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM60-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM60-NEXT: shl.b32 %r69, %r68, 16;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM60-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM60-NEXT: shl.b32 %r71, %r70, 16;
+; SM60-NEXT: sub.rn.f32 %r72, %r71, %r69;
+; SM60-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM60-NEXT: add.s32 %r74, %r73, %r72;
+; SM60-NEXT: add.s32 %r75, %r74, 32767;
+; SM60-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM60-NEXT: or.b32 %r76, %r72, 4194304;
+; SM60-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM60-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM60-NEXT: shl.b32 %r79, %r78, 16;
+; SM60-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM60-NEXT: shl.b32 %r81, %r80, 16;
+; SM60-NEXT: sub.rn.f32 %r82, %r81, %r79;
+; SM60-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM60-NEXT: add.s32 %r84, %r83, %r82;
+; SM60-NEXT: add.s32 %r85, %r84, 32767;
+; SM60-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM60-NEXT: or.b32 %r86, %r82, 4194304;
+; SM60-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM60-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB239_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM60-LABEL: fmin_acq_rel_v2bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r24, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: $L__BB240_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: min.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: min.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM60-NEXT: mov.b32 %r24, %r1;
+; SM60-NEXT: @%p3 bra $L__BB240_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM60-LABEL: fmin_acq_rel_v4bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<47>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
+; SM60-NEXT: $L__BB241_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: min.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: min.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: min.f32 %r28, %r27, %r25;
+; SM60-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM60-NEXT: add.s32 %r30, %r29, %r28;
+; SM60-NEXT: add.s32 %r31, %r30, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM60-NEXT: or.b32 %r32, %r28, 4194304;
+; SM60-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: min.f32 %r38, %r37, %r35;
+; SM60-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM60-NEXT: add.s32 %r40, %r39, %r38;
+; SM60-NEXT: add.s32 %r41, %r40, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM60-NEXT: or.b32 %r42, %r38, 4194304;
+; SM60-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM60-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB241_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM60-LABEL: fmin_acq_rel_v8bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<93>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: $L__BB242_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: min.f32 %r9, %r8, %r6;
+; SM60-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM60-NEXT: add.s32 %r11, %r10, %r9;
+; SM60-NEXT: add.s32 %r12, %r11, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM60-NEXT: or.b32 %r13, %r9, 4194304;
+; SM60-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM60-NEXT: shl.b32 %r18, %r17, 16;
+; SM60-NEXT: min.f32 %r19, %r18, %r16;
+; SM60-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM60-NEXT: add.s32 %r21, %r20, %r19;
+; SM60-NEXT: add.s32 %r22, %r21, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM60-NEXT: or.b32 %r23, %r19, 4194304;
+; SM60-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM60-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM60-NEXT: shl.b32 %r29, %r28, 16;
+; SM60-NEXT: min.f32 %r30, %r29, %r27;
+; SM60-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM60-NEXT: add.s32 %r32, %r31, %r30;
+; SM60-NEXT: add.s32 %r33, %r32, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM60-NEXT: or.b32 %r34, %r30, 4194304;
+; SM60-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM60-NEXT: shl.b32 %r39, %r38, 16;
+; SM60-NEXT: min.f32 %r40, %r39, %r37;
+; SM60-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM60-NEXT: add.s32 %r42, %r41, %r40;
+; SM60-NEXT: add.s32 %r43, %r42, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM60-NEXT: or.b32 %r44, %r40, 4194304;
+; SM60-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM60-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB242_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM60-NEXT: $L__BB242_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM60-NEXT: shl.b32 %r48, %r47, 16;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM60-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM60-NEXT: shl.b32 %r50, %r49, 16;
+; SM60-NEXT: min.f32 %r51, %r50, %r48;
+; SM60-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM60-NEXT: add.s32 %r53, %r52, %r51;
+; SM60-NEXT: add.s32 %r54, %r53, 32767;
+; SM60-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM60-NEXT: or.b32 %r55, %r51, 4194304;
+; SM60-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM60-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM60-NEXT: shl.b32 %r58, %r57, 16;
+; SM60-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM60-NEXT: shl.b32 %r60, %r59, 16;
+; SM60-NEXT: min.f32 %r61, %r60, %r58;
+; SM60-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM60-NEXT: add.s32 %r63, %r62, %r61;
+; SM60-NEXT: add.s32 %r64, %r63, 32767;
+; SM60-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM60-NEXT: or.b32 %r65, %r61, 4194304;
+; SM60-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM60-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM60-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM60-NEXT: shl.b32 %r69, %r68, 16;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM60-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM60-NEXT: shl.b32 %r71, %r70, 16;
+; SM60-NEXT: min.f32 %r72, %r71, %r69;
+; SM60-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM60-NEXT: add.s32 %r74, %r73, %r72;
+; SM60-NEXT: add.s32 %r75, %r74, 32767;
+; SM60-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM60-NEXT: or.b32 %r76, %r72, 4194304;
+; SM60-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM60-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM60-NEXT: shl.b32 %r79, %r78, 16;
+; SM60-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM60-NEXT: shl.b32 %r81, %r80, 16;
+; SM60-NEXT: min.f32 %r82, %r81, %r79;
+; SM60-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM60-NEXT: add.s32 %r84, %r83, %r82;
+; SM60-NEXT: add.s32 %r85, %r84, 32767;
+; SM60-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM60-NEXT: or.b32 %r86, %r82, 4194304;
+; SM60-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM60-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB242_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM60-LABEL: fmax_acq_rel_v2bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r24, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: $L__BB243_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: max.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: max.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM60-NEXT: mov.b32 %r24, %r1;
+; SM60-NEXT: @%p3 bra $L__BB243_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM60-LABEL: fmax_acq_rel_v4bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<47>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
+; SM60-NEXT: $L__BB244_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: max.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: max.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: max.f32 %r28, %r27, %r25;
+; SM60-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM60-NEXT: add.s32 %r30, %r29, %r28;
+; SM60-NEXT: add.s32 %r31, %r30, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM60-NEXT: or.b32 %r32, %r28, 4194304;
+; SM60-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: max.f32 %r38, %r37, %r35;
+; SM60-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM60-NEXT: add.s32 %r40, %r39, %r38;
+; SM60-NEXT: add.s32 %r41, %r40, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM60-NEXT: or.b32 %r42, %r38, 4194304;
+; SM60-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM60-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB244_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM60-LABEL: fmax_acq_rel_v8bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<93>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: $L__BB245_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: max.f32 %r9, %r8, %r6;
+; SM60-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM60-NEXT: add.s32 %r11, %r10, %r9;
+; SM60-NEXT: add.s32 %r12, %r11, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM60-NEXT: or.b32 %r13, %r9, 4194304;
+; SM60-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM60-NEXT: shl.b32 %r18, %r17, 16;
+; SM60-NEXT: max.f32 %r19, %r18, %r16;
+; SM60-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM60-NEXT: add.s32 %r21, %r20, %r19;
+; SM60-NEXT: add.s32 %r22, %r21, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM60-NEXT: or.b32 %r23, %r19, 4194304;
+; SM60-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM60-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM60-NEXT: shl.b32 %r29, %r28, 16;
+; SM60-NEXT: max.f32 %r30, %r29, %r27;
+; SM60-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM60-NEXT: add.s32 %r32, %r31, %r30;
+; SM60-NEXT: add.s32 %r33, %r32, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM60-NEXT: or.b32 %r34, %r30, 4194304;
+; SM60-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM60-NEXT: shl.b32 %r39, %r38, 16;
+; SM60-NEXT: max.f32 %r40, %r39, %r37;
+; SM60-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM60-NEXT: add.s32 %r42, %r41, %r40;
+; SM60-NEXT: add.s32 %r43, %r42, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM60-NEXT: or.b32 %r44, %r40, 4194304;
+; SM60-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM60-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB245_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM60-NEXT: $L__BB245_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM60-NEXT: shl.b32 %r48, %r47, 16;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM60-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM60-NEXT: shl.b32 %r50, %r49, 16;
+; SM60-NEXT: max.f32 %r51, %r50, %r48;
+; SM60-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM60-NEXT: add.s32 %r53, %r52, %r51;
+; SM60-NEXT: add.s32 %r54, %r53, 32767;
+; SM60-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM60-NEXT: or.b32 %r55, %r51, 4194304;
+; SM60-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM60-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM60-NEXT: shl.b32 %r58, %r57, 16;
+; SM60-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM60-NEXT: shl.b32 %r60, %r59, 16;
+; SM60-NEXT: max.f32 %r61, %r60, %r58;
+; SM60-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM60-NEXT: add.s32 %r63, %r62, %r61;
+; SM60-NEXT: add.s32 %r64, %r63, 32767;
+; SM60-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM60-NEXT: or.b32 %r65, %r61, 4194304;
+; SM60-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM60-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM60-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM60-NEXT: shl.b32 %r69, %r68, 16;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM60-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM60-NEXT: shl.b32 %r71, %r70, 16;
+; SM60-NEXT: max.f32 %r72, %r71, %r69;
+; SM60-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM60-NEXT: add.s32 %r74, %r73, %r72;
+; SM60-NEXT: add.s32 %r75, %r74, 32767;
+; SM60-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM60-NEXT: or.b32 %r76, %r72, 4194304;
+; SM60-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM60-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM60-NEXT: shl.b32 %r79, %r78, 16;
+; SM60-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM60-NEXT: shl.b32 %r81, %r80, 16;
+; SM60-NEXT: max.f32 %r82, %r81, %r79;
+; SM60-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM60-NEXT: add.s32 %r84, %r83, %r82;
+; SM60-NEXT: add.s32 %r85, %r84, 32767;
+; SM60-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM60-NEXT: or.b32 %r86, %r82, 4194304;
+; SM60-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM60-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB245_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM60-LABEL: fminimum_acq_rel_v2bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<12>;
+; SM60-NEXT: .reg .b16 %rs<15>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM60-NEXT: $L__BB246_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r16;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: setp.lt.f32 %p1, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM60-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NEXT: setp.lt.f32 %p6, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM60-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
+; SM60-NEXT: setp.ne.b32 %p11, %r1, %r16;
+; SM60-NEXT: mov.b32 %r16, %r1;
+; SM60-NEXT: @%p11 bra $L__BB246_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM60-LABEL: fminimum_acq_rel_v4bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<22>;
+; SM60-NEXT: .reg .b16 %rs<29>;
+; SM60-NEXT: .reg .b32 %r<31>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM60-NEXT: shl.b32 %r17, %r16, 16;
+; SM60-NEXT: setp.eq.b16 %p14, %rs16, -32768;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs15;
+; SM60-NEXT: shl.b32 %r23, %r22, 16;
+; SM60-NEXT: setp.eq.b16 %p19, %rs15, -32768;
+; SM60-NEXT: $L__BB247_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r29;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: setp.lt.f32 %p1, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM60-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NEXT: setp.lt.f32 %p6, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM60-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r15;
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r30;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs18;
+; SM60-NEXT: shl.b32 %r19, %r18, 16;
+; SM60-NEXT: setp.lt.f32 %p11, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM60-NEXT: setp.nan.f32 %p12, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs18, -32768;
+; SM60-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM60-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM60-NEXT: shl.b32 %r21, %r20, 16;
+; SM60-NEXT: setp.eq.f32 %p15, %r21, 0f00000000;
+; SM60-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs17;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: setp.lt.f32 %p16, %r25, %r23;
+; SM60-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM60-NEXT: setp.nan.f32 %p17, %r25, %r23;
+; SM60-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs17, -32768;
+; SM60-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM60-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs25;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: setp.eq.f32 %p20, %r27, 0f00000000;
+; SM60-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM60-NEXT: mov.b32 %r28, {%rs28, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r28;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r29;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r30;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB247_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM60-LABEL: fminimum_acq_rel_v8bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<43>;
+; SM60-NEXT: .reg .b16 %rs<57>;
+; SM60-NEXT: .reg .b32 %r<61>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r57, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs16;
+; SM60-NEXT: shl.b32 %r19, %r18, 16;
+; SM60-NEXT: setp.eq.b16 %p14, %rs16, -32768;
+; SM60-NEXT: setp.eq.b16 %p19, %rs15, -32768;
+; SM60-NEXT: $L__BB248_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r57;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: setp.lt.f32 %p1, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r10, 0f00000000;
+; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs1;
+; SM60-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs3;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: setp.lt.f32 %p6, %r14, %r12;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f32 %p7, %r14, %r12;
+; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs11;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: setp.eq.f32 %p10, %r16, 0f00000000;
+; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM60-NEXT: mov.b32 %r17, {%rs14, %rs9};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r58;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM60-NEXT: shl.b32 %r21, %r20, 16;
+; SM60-NEXT: setp.lt.f32 %p11, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM60-NEXT: setp.nan.f32 %p12, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs18, -32768;
+; SM60-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM60-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs20;
+; SM60-NEXT: shl.b32 %r23, %r22, 16;
+; SM60-NEXT: setp.eq.f32 %p15, %r23, 0f00000000;
+; SM60-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs15;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs17;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: setp.lt.f32 %p16, %r27, %r25;
+; SM60-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM60-NEXT: setp.nan.f32 %p17, %r27, %r25;
+; SM60-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs17, -32768;
+; SM60-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM60-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM60-NEXT: cvt.u32.u16 %r28, %rs25;
+; SM60-NEXT: shl.b32 %r29, %r28, 16;
+; SM60-NEXT: setp.eq.f32 %p20, %r29, 0f00000000;
+; SM60-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM60-NEXT: mov.b32 %r30, {%rs28, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r30;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r57;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r58;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r57, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB248_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r59, %rd12;
+; SM60-NEXT: $L__BB248_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs29, %rs30}, %r3;
+; SM60-NEXT: cvt.u32.u16 %r31, %rs30;
+; SM60-NEXT: shl.b32 %r32, %r31, 16;
+; SM60-NEXT: mov.b32 {%rs31, %rs32}, %r59;
+; SM60-NEXT: cvt.u32.u16 %r33, %rs32;
+; SM60-NEXT: shl.b32 %r34, %r33, 16;
+; SM60-NEXT: setp.lt.f32 %p22, %r34, %r32;
+; SM60-NEXT: selp.b16 %rs33, %rs32, %rs30, %p22;
+; SM60-NEXT: setp.nan.f32 %p23, %r34, %r32;
+; SM60-NEXT: selp.b16 %rs34, 0x7FC0, %rs33, %p23;
+; SM60-NEXT: setp.eq.b16 %p24, %rs32, -32768;
+; SM60-NEXT: selp.b16 %rs35, %rs32, %rs34, %p24;
+; SM60-NEXT: setp.eq.b16 %p25, %rs30, -32768;
+; SM60-NEXT: selp.b16 %rs36, %rs30, %rs35, %p25;
+; SM60-NEXT: cvt.u32.u16 %r35, %rs34;
+; SM60-NEXT: shl.b32 %r36, %r35, 16;
+; SM60-NEXT: setp.eq.f32 %p26, %r36, 0f00000000;
+; SM60-NEXT: selp.b16 %rs37, %rs36, %rs34, %p26;
+; SM60-NEXT: cvt.u32.u16 %r37, %rs29;
+; SM60-NEXT: shl.b32 %r38, %r37, 16;
+; SM60-NEXT: cvt.u32.u16 %r39, %rs31;
+; SM60-NEXT: shl.b32 %r40, %r39, 16;
+; SM60-NEXT: setp.lt.f32 %p27, %r40, %r38;
+; SM60-NEXT: selp.b16 %rs38, %rs31, %rs29, %p27;
+; SM60-NEXT: setp.nan.f32 %p28, %r40, %r38;
+; SM60-NEXT: selp.b16 %rs39, 0x7FC0, %rs38, %p28;
+; SM60-NEXT: setp.eq.b16 %p29, %rs31, -32768;
+; SM60-NEXT: selp.b16 %rs40, %rs31, %rs39, %p29;
+; SM60-NEXT: setp.eq.b16 %p30, %rs29, -32768;
+; SM60-NEXT: selp.b16 %rs41, %rs29, %rs40, %p30;
+; SM60-NEXT: cvt.u32.u16 %r41, %rs39;
+; SM60-NEXT: shl.b32 %r42, %r41, 16;
+; SM60-NEXT: setp.eq.f32 %p31, %r42, 0f00000000;
+; SM60-NEXT: selp.b16 %rs42, %rs41, %rs39, %p31;
+; SM60-NEXT: mov.b32 %r43, {%rs42, %rs37};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r43;
+; SM60-NEXT: mov.b32 {%rs43, %rs44}, %r4;
+; SM60-NEXT: cvt.u32.u16 %r44, %rs44;
+; SM60-NEXT: shl.b32 %r45, %r44, 16;
+; SM60-NEXT: mov.b32 {%rs45, %rs46}, %r60;
+; SM60-NEXT: cvt.u32.u16 %r46, %rs46;
+; SM60-NEXT: shl.b32 %r47, %r46, 16;
+; SM60-NEXT: setp.lt.f32 %p32, %r47, %r45;
+; SM60-NEXT: selp.b16 %rs47, %rs46, %rs44, %p32;
+; SM60-NEXT: setp.nan.f32 %p33, %r47, %r45;
+; SM60-NEXT: selp.b16 %rs48, 0x7FC0, %rs47, %p33;
+; SM60-NEXT: setp.eq.b16 %p34, %rs46, -32768;
+; SM60-NEXT: selp.b16 %rs49, %rs46, %rs48, %p34;
+; SM60-NEXT: setp.eq.b16 %p35, %rs44, -32768;
+; SM60-NEXT: selp.b16 %rs50, %rs44, %rs49, %p35;
+; SM60-NEXT: cvt.u32.u16 %r48, %rs48;
+; SM60-NEXT: shl.b32 %r49, %r48, 16;
+; SM60-NEXT: setp.eq.f32 %p36, %r49, 0f00000000;
+; SM60-NEXT: selp.b16 %rs51, %rs50, %rs48, %p36;
+; SM60-NEXT: cvt.u32.u16 %r50, %rs43;
+; SM60-NEXT: shl.b32 %r51, %r50, 16;
+; SM60-NEXT: cvt.u32.u16 %r52, %rs45;
+; SM60-NEXT: shl.b32 %r53, %r52, 16;
+; SM60-NEXT: setp.lt.f32 %p37, %r53, %r51;
+; SM60-NEXT: selp.b16 %rs52, %rs45, %rs43, %p37;
+; SM60-NEXT: setp.nan.f32 %p38, %r53, %r51;
+; SM60-NEXT: selp.b16 %rs53, 0x7FC0, %rs52, %p38;
+; SM60-NEXT: setp.eq.b16 %p39, %rs45, -32768;
+; SM60-NEXT: selp.b16 %rs54, %rs45, %rs53, %p39;
+; SM60-NEXT: setp.eq.b16 %p40, %rs43, -32768;
+; SM60-NEXT: selp.b16 %rs55, %rs43, %rs54, %p40;
+; SM60-NEXT: cvt.u32.u16 %r54, %rs53;
+; SM60-NEXT: shl.b32 %r55, %r54, 16;
+; SM60-NEXT: setp.eq.f32 %p41, %r55, 0f00000000;
+; SM60-NEXT: selp.b16 %rs56, %rs55, %rs53, %p41;
+; SM60-NEXT: mov.b32 %r56, {%rs56, %rs51};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r56;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r59;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r60;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r59, %rd21;
+; SM60-NEXT: @%p42 bra $L__BB248_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<12>;
+; SM60-NEXT: .reg .b16 %rs<15>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM60-NEXT: $L__BB249_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r16;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: setp.gt.f32 %p1, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM60-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NEXT: setp.gt.f32 %p6, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM60-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
+; SM60-NEXT: setp.ne.b32 %p11, %r1, %r16;
+; SM60-NEXT: mov.b32 %r16, %r1;
+; SM60-NEXT: @%p11 bra $L__BB249_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v4bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<22>;
+; SM60-NEXT: .reg .b16 %rs<29>;
+; SM60-NEXT: .reg .b32 %r<31>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM60-NEXT: shl.b32 %r17, %r16, 16;
+; SM60-NEXT: setp.eq.b16 %p14, %rs16, 0;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs15;
+; SM60-NEXT: shl.b32 %r23, %r22, 16;
+; SM60-NEXT: setp.eq.b16 %p19, %rs15, 0;
+; SM60-NEXT: $L__BB250_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r29;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: setp.gt.f32 %p1, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM60-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NEXT: setp.gt.f32 %p6, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM60-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r15;
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r30;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs18;
+; SM60-NEXT: shl.b32 %r19, %r18, 16;
+; SM60-NEXT: setp.gt.f32 %p11, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM60-NEXT: setp.nan.f32 %p12, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs18, 0;
+; SM60-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM60-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM60-NEXT: shl.b32 %r21, %r20, 16;
+; SM60-NEXT: setp.eq.f32 %p15, %r21, 0f00000000;
+; SM60-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs17;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: setp.gt.f32 %p16, %r25, %r23;
+; SM60-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM60-NEXT: setp.nan.f32 %p17, %r25, %r23;
+; SM60-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs17, 0;
+; SM60-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM60-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs25;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: setp.eq.f32 %p20, %r27, 0f00000000;
+; SM60-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM60-NEXT: mov.b32 %r28, {%rs28, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r28;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r29;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r30;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB250_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v8bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<43>;
+; SM60-NEXT: .reg .b16 %rs<57>;
+; SM60-NEXT: .reg .b32 %r<61>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r57, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs16;
+; SM60-NEXT: shl.b32 %r19, %r18, 16;
+; SM60-NEXT: setp.eq.b16 %p14, %rs16, 0;
+; SM60-NEXT: setp.eq.b16 %p19, %rs15, 0;
+; SM60-NEXT: $L__BB251_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r57;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: setp.gt.f32 %p1, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r10, 0f00000000;
+; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs1;
+; SM60-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs3;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: setp.gt.f32 %p6, %r14, %r12;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f32 %p7, %r14, %r12;
+; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs11;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: setp.eq.f32 %p10, %r16, 0f00000000;
+; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM60-NEXT: mov.b32 %r17, {%rs14, %rs9};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r58;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM60-NEXT: shl.b32 %r21, %r20, 16;
+; SM60-NEXT: setp.gt.f32 %p11, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM60-NEXT: setp.nan.f32 %p12, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs18, 0;
+; SM60-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM60-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs20;
+; SM60-NEXT: shl.b32 %r23, %r22, 16;
+; SM60-NEXT: setp.eq.f32 %p15, %r23, 0f00000000;
+; SM60-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs15;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs17;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: setp.gt.f32 %p16, %r27, %r25;
+; SM60-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM60-NEXT: setp.nan.f32 %p17, %r27, %r25;
+; SM60-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs17, 0;
+; SM60-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM60-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM60-NEXT: cvt.u32.u16 %r28, %rs25;
+; SM60-NEXT: shl.b32 %r29, %r28, 16;
+; SM60-NEXT: setp.eq.f32 %p20, %r29, 0f00000000;
+; SM60-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM60-NEXT: mov.b32 %r30, {%rs28, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r30;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r57;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r58;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r57, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB251_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r59, %rd12;
+; SM60-NEXT: $L__BB251_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs29, %rs30}, %r3;
+; SM60-NEXT: cvt.u32.u16 %r31, %rs30;
+; SM60-NEXT: shl.b32 %r32, %r31, 16;
+; SM60-NEXT: mov.b32 {%rs31, %rs32}, %r59;
+; SM60-NEXT: cvt.u32.u16 %r33, %rs32;
+; SM60-NEXT: shl.b32 %r34, %r33, 16;
+; SM60-NEXT: setp.gt.f32 %p22, %r34, %r32;
+; SM60-NEXT: selp.b16 %rs33, %rs32, %rs30, %p22;
+; SM60-NEXT: setp.nan.f32 %p23, %r34, %r32;
+; SM60-NEXT: selp.b16 %rs34, 0x7FC0, %rs33, %p23;
+; SM60-NEXT: setp.eq.b16 %p24, %rs32, 0;
+; SM60-NEXT: selp.b16 %rs35, %rs32, %rs34, %p24;
+; SM60-NEXT: setp.eq.b16 %p25, %rs30, 0;
+; SM60-NEXT: selp.b16 %rs36, %rs30, %rs35, %p25;
+; SM60-NEXT: cvt.u32.u16 %r35, %rs34;
+; SM60-NEXT: shl.b32 %r36, %r35, 16;
+; SM60-NEXT: setp.eq.f32 %p26, %r36, 0f00000000;
+; SM60-NEXT: selp.b16 %rs37, %rs36, %rs34, %p26;
+; SM60-NEXT: cvt.u32.u16 %r37, %rs29;
+; SM60-NEXT: shl.b32 %r38, %r37, 16;
+; SM60-NEXT: cvt.u32.u16 %r39, %rs31;
+; SM60-NEXT: shl.b32 %r40, %r39, 16;
+; SM60-NEXT: setp.gt.f32 %p27, %r40, %r38;
+; SM60-NEXT: selp.b16 %rs38, %rs31, %rs29, %p27;
+; SM60-NEXT: setp.nan.f32 %p28, %r40, %r38;
+; SM60-NEXT: selp.b16 %rs39, 0x7FC0, %rs38, %p28;
+; SM60-NEXT: setp.eq.b16 %p29, %rs31, 0;
+; SM60-NEXT: selp.b16 %rs40, %rs31, %rs39, %p29;
+; SM60-NEXT: setp.eq.b16 %p30, %rs29, 0;
+; SM60-NEXT: selp.b16 %rs41, %rs29, %rs40, %p30;
+; SM60-NEXT: cvt.u32.u16 %r41, %rs39;
+; SM60-NEXT: shl.b32 %r42, %r41, 16;
+; SM60-NEXT: setp.eq.f32 %p31, %r42, 0f00000000;
+; SM60-NEXT: selp.b16 %rs42, %rs41, %rs39, %p31;
+; SM60-NEXT: mov.b32 %r43, {%rs42, %rs37};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r43;
+; SM60-NEXT: mov.b32 {%rs43, %rs44}, %r4;
+; SM60-NEXT: cvt.u32.u16 %r44, %rs44;
+; SM60-NEXT: shl.b32 %r45, %r44, 16;
+; SM60-NEXT: mov.b32 {%rs45, %rs46}, %r60;
+; SM60-NEXT: cvt.u32.u16 %r46, %rs46;
+; SM60-NEXT: shl.b32 %r47, %r46, 16;
+; SM60-NEXT: setp.gt.f32 %p32, %r47, %r45;
+; SM60-NEXT: selp.b16 %rs47, %rs46, %rs44, %p32;
+; SM60-NEXT: setp.nan.f32 %p33, %r47, %r45;
+; SM60-NEXT: selp.b16 %rs48, 0x7FC0, %rs47, %p33;
+; SM60-NEXT: setp.eq.b16 %p34, %rs46, 0;
+; SM60-NEXT: selp.b16 %rs49, %rs46, %rs48, %p34;
+; SM60-NEXT: setp.eq.b16 %p35, %rs44, 0;
+; SM60-NEXT: selp.b16 %rs50, %rs44, %rs49, %p35;
+; SM60-NEXT: cvt.u32.u16 %r48, %rs48;
+; SM60-NEXT: shl.b32 %r49, %r48, 16;
+; SM60-NEXT: setp.eq.f32 %p36, %r49, 0f00000000;
+; SM60-NEXT: selp.b16 %rs51, %rs50, %rs48, %p36;
+; SM60-NEXT: cvt.u32.u16 %r50, %rs43;
+; SM60-NEXT: shl.b32 %r51, %r50, 16;
+; SM60-NEXT: cvt.u32.u16 %r52, %rs45;
+; SM60-NEXT: shl.b32 %r53, %r52, 16;
+; SM60-NEXT: setp.gt.f32 %p37, %r53, %r51;
+; SM60-NEXT: selp.b16 %rs52, %rs45, %rs43, %p37;
+; SM60-NEXT: setp.nan.f32 %p38, %r53, %r51;
+; SM60-NEXT: selp.b16 %rs53, 0x7FC0, %rs52, %p38;
+; SM60-NEXT: setp.eq.b16 %p39, %rs45, 0;
+; SM60-NEXT: selp.b16 %rs54, %rs45, %rs53, %p39;
+; SM60-NEXT: setp.eq.b16 %p40, %rs43, 0;
+; SM60-NEXT: selp.b16 %rs55, %rs43, %rs54, %p40;
+; SM60-NEXT: cvt.u32.u16 %r54, %rs53;
+; SM60-NEXT: shl.b32 %r55, %r54, 16;
+; SM60-NEXT: setp.eq.f32 %p41, %r55, 0f00000000;
+; SM60-NEXT: selp.b16 %rs56, %rs55, %rs53, %p41;
+; SM60-NEXT: mov.b32 %r56, {%rs56, %rs51};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r56;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r59;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r60;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r59, %rd21;
+; SM60-NEXT: @%p42 bra $L__BB251_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_monotonic_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_monotonic_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [add_monotonic_v2i8_global_cta_param_0];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB252_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB252_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_acquire_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB253_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB253_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_release_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB254_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB254_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_seq_cst_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB255_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB255_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: add_monotonic_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_monotonic_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: $L__BB256_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB256_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: add_acquire_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: $L__BB257_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB257_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: add_release_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: $L__BB258_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB258_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: add_seq_cst_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: $L__BB259_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB259_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: add_monotonic_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB260_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM60-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM60-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM60-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB260_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: add_acquire_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB261_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM60-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM60-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM60-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB261_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: add_release_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB262_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM60-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM60-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM60-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB262_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: add_seq_cst_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB263_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM60-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM60-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM60-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB263_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
+}
+
+define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: add_monotonic_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v2i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: add_acquire_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: add_release_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: add_seq_cst_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_monotonic_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_acquire_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_release_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_seq_cst_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: add_monotonic_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: add_acquire_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: add_release_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_release_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: add_seq_cst_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_seq_cst_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
+ ret <8 x i32> %retval
+}
+
+define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: nand_monotonic_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_monotonic_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [nand_monotonic_v2i8_global_cta_param_0];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: $L__BB276_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r18, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, %r4;
+; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM60-NEXT: shl.b32 %r14, %r13, %r1;
+; SM60-NEXT: and.b32 %r15, %r18, %r2;
+; SM60-NEXT: or.b32 %r16, %r15, %r14;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: @%p1 bra $L__BB276_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r17, %r3, %r1;
+; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: nand_acquire_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acquire_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [nand_acquire_v2i8_global_cta_param_0];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: $L__BB277_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r18, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, %r4;
+; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM60-NEXT: shl.b32 %r14, %r13, %r1;
+; SM60-NEXT: and.b32 %r15, %r18, %r2;
+; SM60-NEXT: or.b32 %r16, %r15, %r14;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: @%p1 bra $L__BB277_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r17, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: nand_release_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: $L__BB278_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r18, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, %r4;
+; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM60-NEXT: shl.b32 %r14, %r13, %r1;
+; SM60-NEXT: and.b32 %r15, %r18, %r2;
+; SM60-NEXT: or.b32 %r16, %r15, %r14;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: @%p1 bra $L__BB278_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r17, %r3, %r1;
+; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: nand_seq_cst_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: $L__BB279_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r18, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, %r4;
+; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM60-NEXT: shl.b32 %r14, %r13, %r1;
+; SM60-NEXT: and.b32 %r15, %r18, %r2;
+; SM60-NEXT: or.b32 %r16, %r15, %r14;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: @%p1 bra $L__BB279_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r17, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_monotonic_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB280_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB280_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_acquire_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB281_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB281_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_release_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB282_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB282_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_seq_cst_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_seq_cst_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB283_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB283_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_monotonic_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB284_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB284_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_acquire_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB285_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB285_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_release_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB286_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB286_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_seq_cst_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB287_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB287_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
+}
+
+define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_monotonic_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB288_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB288_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_acquire_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB289_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB289_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_release_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB290_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB290_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_seq_cst_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB291_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB291_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_monotonic_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB292_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r10, %r2;
+; SM60-NEXT: and.b32 %r6, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB292_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM60-NEXT: $L__BB292_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r7, %r12, %r4;
+; SM60-NEXT: and.b32 %r8, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB292_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_acquire_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB293_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r10, %r2;
+; SM60-NEXT: and.b32 %r6, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB293_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM60-NEXT: $L__BB293_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r7, %r12, %r4;
+; SM60-NEXT: and.b32 %r8, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB293_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_release_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB294_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r10, %r2;
+; SM60-NEXT: and.b32 %r6, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB294_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM60-NEXT: $L__BB294_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r7, %r12, %r4;
+; SM60-NEXT: and.b32 %r8, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB294_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_seq_cst_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB295_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r10, %r2;
+; SM60-NEXT: and.b32 %r6, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB295_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM60-NEXT: $L__BB295_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r7, %r12, %r4;
+; SM60-NEXT: and.b32 %r8, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB295_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: nand_monotonic_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<46>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_monotonic_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB296_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r9, %r18, %r6;
+; SM60-NEXT: and.b32 %r10, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB296_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM60-NEXT: $L__BB296_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r11, %r20, %r8;
+; SM60-NEXT: and.b32 %r12, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB296_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM60-NEXT: $L__BB296_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM60-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM60-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM60-NEXT: and.b32 %r13, %r22, %r2;
+; SM60-NEXT: and.b32 %r14, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM60-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM60-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM60-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM60-NEXT: not.b64 %rd33, %rd32;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM60-NEXT: @%p3 bra $L__BB296_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM60-NEXT: $L__BB296_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM60-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM60-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM60-NEXT: and.b32 %r15, %r24, %r4;
+; SM60-NEXT: and.b32 %r16, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM60-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM60-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM60-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM60-NEXT: not.b64 %rd44, %rd43;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM60-NEXT: @%p4 bra $L__BB296_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: nand_acquire_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<46>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acquire_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r9, %r18, %r6;
+; SM60-NEXT: and.b32 %r10, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB297_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM60-NEXT: $L__BB297_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r11, %r20, %r8;
+; SM60-NEXT: and.b32 %r12, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB297_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM60-NEXT: $L__BB297_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM60-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM60-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM60-NEXT: and.b32 %r13, %r22, %r2;
+; SM60-NEXT: and.b32 %r14, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM60-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM60-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM60-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM60-NEXT: not.b64 %rd33, %rd32;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM60-NEXT: @%p3 bra $L__BB297_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM60-NEXT: $L__BB297_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM60-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM60-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM60-NEXT: and.b32 %r15, %r24, %r4;
+; SM60-NEXT: and.b32 %r16, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM60-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM60-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM60-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM60-NEXT: not.b64 %rd44, %rd43;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM60-NEXT: @%p4 bra $L__BB297_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: nand_release_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<46>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_release_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB298_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r9, %r18, %r6;
+; SM60-NEXT: and.b32 %r10, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB298_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM60-NEXT: $L__BB298_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r11, %r20, %r8;
+; SM60-NEXT: and.b32 %r12, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB298_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM60-NEXT: $L__BB298_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM60-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM60-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM60-NEXT: and.b32 %r13, %r22, %r2;
+; SM60-NEXT: and.b32 %r14, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM60-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM60-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM60-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM60-NEXT: not.b64 %rd33, %rd32;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM60-NEXT: @%p3 bra $L__BB298_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM60-NEXT: $L__BB298_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM60-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM60-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM60-NEXT: and.b32 %r15, %r24, %r4;
+; SM60-NEXT: and.b32 %r16, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM60-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM60-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM60-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM60-NEXT: not.b64 %rd44, %rd43;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM60-NEXT: @%p4 bra $L__BB298_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: nand_seq_cst_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<46>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_seq_cst_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB299_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r9, %r18, %r6;
+; SM60-NEXT: and.b32 %r10, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB299_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM60-NEXT: $L__BB299_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r11, %r20, %r8;
+; SM60-NEXT: and.b32 %r12, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB299_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM60-NEXT: $L__BB299_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM60-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM60-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM60-NEXT: and.b32 %r13, %r22, %r2;
+; SM60-NEXT: and.b32 %r14, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM60-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM60-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM60-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM60-NEXT: not.b64 %rd33, %rd32;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM60-NEXT: @%p3 bra $L__BB299_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM60-NEXT: $L__BB299_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM60-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM60-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM60-NEXT: and.b32 %r15, %r24, %r4;
+; SM60-NEXT: and.b32 %r16, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM60-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM60-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM60-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM60-NEXT: not.b64 %rd44, %rd43;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM60-NEXT: @%p4 bra $L__BB299_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
+ ret <8 x i32> %retval
+}
+
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
new file mode 100644
index 0000000000000..e2f3877911f15
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
@@ -0,0 +1,16076 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s --check-prefix=SM70
+; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | %ptxas-verify -arch=sm_70 %}
+
+define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: xchg_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<14>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
+; SM70-NEXT: mov.b32 %r8, 65535;
+; SM70-NEXT: shl.b32 %r9, %r8, %r1;
+; SM70-NEXT: not.b32 %r2, %r9;
+; SM70-NEXT: shl.b32 %r3, %r5, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM70-NEXT: $L__BB0_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r13, %r2;
+; SM70-NEXT: or.b32 %r11, %r10, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM70-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM70-NEXT: mov.b32 %r13, %r4;
+; SM70-NEXT: @%p1 bra $L__BB0_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r12, %r4, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r12;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @xchg_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: xchg_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @xchg_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: xchg_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @xchg_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: xchg_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @xchg_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: xchg_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @xchg_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: xchg_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: cvt.u64.u32 %rd2, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r2;
+; SM70-NEXT: shl.b64 %rd4, %rd3, 32;
+; SM70-NEXT: or.b64 %rd5, %rd2, %rd4;
+; SM70-NEXT: cvt.u64.u32 %rd6, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: shl.b64 %rd8, %rd7, 32;
+; SM70-NEXT: or.b64 %rd9, %rd6, %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd10, [%rd1], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @xchg_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: xchg_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [xchg_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @xchg_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: xchg_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @xchg_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: xchg_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xchg_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @xchg_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: xchg_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @xchg_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: xchg_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @xchg_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: xchg_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xchg_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xchg_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [add_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB12_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB12_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: add_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: $L__BB13_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB13_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: add_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB14_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM70-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM70-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM70-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB14_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: add_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB15_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: add.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: add.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB15_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: add_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB16_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: add.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: add.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM70-NEXT: add.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: add.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB16_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: add_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB17_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: add.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: add.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM70-NEXT: add.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: add.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB17_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: $L__BB17_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM70-NEXT: add.s16 %rs17, %rs16, %rs14;
+; SM70-NEXT: add.s16 %rs18, %rs15, %rs13;
+; SM70-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: add.s16 %rs23, %rs22, %rs20;
+; SM70-NEXT: add.s16 %rs24, %rs21, %rs19;
+; SM70-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB17_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @add_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: add_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: add_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @add_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: add_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @add_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: add_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @add_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: add_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [add_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [add_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: sub_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [sub_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB24_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: sub.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: sub.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB24_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: sub_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [sub_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: $L__BB25_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: sub.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: sub.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB25_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: sub_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB26_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: sub.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: sub.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM70-NEXT: sub.s16 %rs15, %rs14, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM70-NEXT: sub.s16 %rs18, %rs17, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM70-NEXT: sub.s16 %rs21, %rs20, %rs19;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM70-NEXT: sub.s16 %rs24, %rs23, %rs22;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB26_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: sub_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [sub_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB27_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB27_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: sub_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB28_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM70-NEXT: sub.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: sub.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB28_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: sub_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB29_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM70-NEXT: sub.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: sub.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB29_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: $L__BB29_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM70-NEXT: sub.s16 %rs17, %rs16, %rs14;
+; SM70-NEXT: sub.s16 %rs18, %rs15, %rs13;
+; SM70-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: sub.s16 %rs23, %rs22, %rs20;
+; SM70-NEXT: sub.s16 %rs24, %rs21, %rs19;
+; SM70-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB29_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: sub_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: neg.s32 %r3, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1], %r3;
+; SM70-NEXT: neg.s32 %r5, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r5;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @sub_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: sub_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: neg.s32 %r5, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1], %r5;
+; SM70-NEXT: neg.s32 %r7, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+4], %r7;
+; SM70-NEXT: neg.s32 %r9, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+8], %r9;
+; SM70-NEXT: neg.s32 %r11, %r4;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r11;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: sub_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: neg.s32 %r9, %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1], %r9;
+; SM70-NEXT: neg.s32 %r11, %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+4], %r11;
+; SM70-NEXT: neg.s32 %r13, %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+8], %r13;
+; SM70-NEXT: neg.s32 %r15, %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+12], %r15;
+; SM70-NEXT: neg.s32 %r17, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r18, [%rd1+16], %r17;
+; SM70-NEXT: neg.s32 %r19, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r20, [%rd1+20], %r19;
+; SM70-NEXT: neg.s32 %r21, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r22, [%rd1+24], %r21;
+; SM70-NEXT: neg.s32 %r23, %r4;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r24, [%rd1+28], %r23;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: sub_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<8>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: neg.s64 %rd4, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd5, [%rd1], %rd4;
+; SM70-NEXT: neg.s64 %rd6, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd6;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @sub_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: sub_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<14>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: neg.s64 %rd6, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1], %rd6;
+; SM70-NEXT: neg.s64 %rd8, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd9, [%rd1+8], %rd8;
+; SM70-NEXT: neg.s64 %rd10, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1+16], %rd10;
+; SM70-NEXT: neg.s64 %rd12, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd12;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd11, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd7, %rd9};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: sub_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<26>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [sub_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: neg.s64 %rd10, %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1], %rd10;
+; SM70-NEXT: neg.s64 %rd12, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+8], %rd12;
+; SM70-NEXT: neg.s64 %rd14, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd15, [%rd1+16], %rd14;
+; SM70-NEXT: neg.s64 %rd16, %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd17, [%rd1+24], %rd16;
+; SM70-NEXT: neg.s64 %rd18, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd19, [%rd1+32], %rd18;
+; SM70-NEXT: neg.s64 %rd20, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd21, [%rd1+40], %rd20;
+; SM70-NEXT: neg.s64 %rd22, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd23, [%rd1+48], %rd22;
+; SM70-NEXT: neg.s64 %rd24, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd25, [%rd1+56], %rd24;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd23, %rd25};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd19, %rd21};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd15, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd11, %rd13};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @and_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: and_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<12>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM70-NEXT: and.b32 %r2, %r1, 3;
+; SM70-NEXT: shl.b32 %r3, %r2, 3;
+; SM70-NEXT: mov.b32 %r4, 65535;
+; SM70-NEXT: shl.b32 %r5, %r4, %r3;
+; SM70-NEXT: not.b32 %r6, %r5;
+; SM70-NEXT: ld.param.b16 %r7, [and_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r8, %r7, %r3;
+; SM70-NEXT: or.b32 %r9, %r8, %r6;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM70-NEXT: shr.u32 %r11, %r10, %r3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r11;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @and_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: and_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [and_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB37_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB37_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: and_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB38_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: and.b32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB38_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: and_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [and_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB39_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB39_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: and_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB40_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: and.b32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB40_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: and_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB41_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: and.b32 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB41_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB41_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: and.b32 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB41_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @and_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: and_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @and_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: and_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @and_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: and_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [and_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @and_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: and_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @and_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: and_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @and_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: and_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [and_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [and_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: nand_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: $L__BB48_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r18, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, %r4;
+; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM70-NEXT: shl.b32 %r14, %r13, %r1;
+; SM70-NEXT: and.b32 %r15, %r18, %r2;
+; SM70-NEXT: or.b32 %r16, %r15, %r14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: @%p1 bra $L__BB48_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r17, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB49_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB49_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB50_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB50_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: nand_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB51_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB51_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: nand_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB52_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB52_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: nand_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM70-NEXT: $L__BB53_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r14, %r2;
+; SM70-NEXT: and.b32 %r6, %r13, %r1;
+; SM70-NEXT: xor.b32 %r7, %r6, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM70-NEXT: xor.b32 %r8, %r5, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB53_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM70-NEXT: $L__BB53_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r15;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r16;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: and.b32 %r9, %r16, %r4;
+; SM70-NEXT: and.b32 %r10, %r15, %r3;
+; SM70-NEXT: xor.b32 %r11, %r10, -1;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: xor.b32 %r12, %r9, -1;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB53_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB54_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB54_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB55_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r10, %r2;
+; SM70-NEXT: and.b32 %r6, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB55_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM70-NEXT: $L__BB55_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r7, %r12, %r4;
+; SM70-NEXT: and.b32 %r8, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB55_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: nand_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<46>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB56_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r9, %r18, %r6;
+; SM70-NEXT: and.b32 %r10, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB56_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM70-NEXT: $L__BB56_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r11, %r20, %r8;
+; SM70-NEXT: and.b32 %r12, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB56_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM70-NEXT: $L__BB56_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM70-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM70-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM70-NEXT: and.b32 %r13, %r22, %r2;
+; SM70-NEXT: and.b32 %r14, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM70-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM70-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM70-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM70-NEXT: not.b64 %rd33, %rd32;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM70-NEXT: @%p3 bra $L__BB56_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM70-NEXT: $L__BB56_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM70-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM70-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM70-NEXT: and.b32 %r15, %r24, %r4;
+; SM70-NEXT: and.b32 %r16, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM70-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM70-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM70-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM70-NEXT: not.b64 %rd44, %rd43;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM70-NEXT: @%p4 bra $L__BB56_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: nand_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [nand_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
+; SM70-NEXT: $L__BB57_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd6, %rd10, %rd1;
+; SM70-NEXT: not.b64 %rd7, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
+; SM70-NEXT: mov.b64 %rd10, %rd3;
+; SM70-NEXT: @%p1 bra $L__BB57_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
+; SM70-NEXT: $L__BB57_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd8, %rd11, %rd2;
+; SM70-NEXT: not.b64 %rd9, %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
+; SM70-NEXT: mov.b64 %rd11, %rd4;
+; SM70-NEXT: @%p2 bra $L__BB57_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: nand_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [nand_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
+; SM70-NEXT: $L__BB58_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd18;
+; SM70-NEXT: and.b64 %rd10, %rd5, %rd3;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
+; SM70-NEXT: @%p1 bra $L__BB58_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
+; SM70-NEXT: $L__BB58_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd19;
+; SM70-NEXT: and.b64 %rd12, %rd6, %rd4;
+; SM70-NEXT: not.b64 %rd13, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM70-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
+; SM70-NEXT: @%p2 bra $L__BB58_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
+; SM70-NEXT: $L__BB58_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd14, %rd20, %rd1;
+; SM70-NEXT: not.b64 %rd15, %rd14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
+; SM70-NEXT: mov.b64 %rd20, %rd7;
+; SM70-NEXT: @%p3 bra $L__BB58_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
+; SM70-NEXT: $L__BB58_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd16, %rd21, %rd2;
+; SM70-NEXT: not.b64 %rd17, %rd16;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
+; SM70-NEXT: mov.b64 %rd21, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB58_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: nand_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [nand_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [nand_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [nand_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
+; SM70-NEXT: $L__BB59_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd34;
+; SM70-NEXT: and.b64 %rd18, %rd7, %rd5;
+; SM70-NEXT: not.b64 %rd19, %rd18;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM70-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
+; SM70-NEXT: @%p1 bra $L__BB59_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
+; SM70-NEXT: $L__BB59_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd35;
+; SM70-NEXT: and.b64 %rd20, %rd8, %rd6;
+; SM70-NEXT: not.b64 %rd21, %rd20;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM70-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
+; SM70-NEXT: @%p2 bra $L__BB59_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
+; SM70-NEXT: $L__BB59_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd36;
+; SM70-NEXT: and.b64 %rd22, %rd9, %rd3;
+; SM70-NEXT: not.b64 %rd23, %rd22;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM70-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
+; SM70-NEXT: @%p3 bra $L__BB59_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
+; SM70-NEXT: $L__BB59_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd37;
+; SM70-NEXT: and.b64 %rd24, %rd10, %rd4;
+; SM70-NEXT: not.b64 %rd25, %rd24;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM70-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
+; SM70-NEXT: @%p4 bra $L__BB59_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
+; SM70-NEXT: $L__BB59_9: // %atomicrmw.start41
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd38;
+; SM70-NEXT: and.b64 %rd26, %rd13, %rd1;
+; SM70-NEXT: not.b64 %rd27, %rd26;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM70-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
+; SM70-NEXT: @%p5 bra $L__BB59_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end40
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
+; SM70-NEXT: $L__BB59_11: // %atomicrmw.start47
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd39;
+; SM70-NEXT: and.b64 %rd28, %rd14, %rd2;
+; SM70-NEXT: not.b64 %rd29, %rd28;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM70-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
+; SM70-NEXT: @%p6 bra $L__BB59_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end46
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
+; SM70-NEXT: $L__BB59_13: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd30, %rd40, %rd11;
+; SM70-NEXT: not.b64 %rd31, %rd30;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
+; SM70-NEXT: mov.b64 %rd40, %rd15;
+; SM70-NEXT: @%p7 bra $L__BB59_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end57
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
+; SM70-NEXT: $L__BB59_15: // %atomicrmw.start64
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd32, %rd41, %rd12;
+; SM70-NEXT: not.b64 %rd33, %rd32;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
+; SM70-NEXT: mov.b64 %rd41, %rd16;
+; SM70-NEXT: @%p8 bra $L__BB59_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end63
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @or_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: or_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM70-NEXT: and.b32 %r2, %r1, 3;
+; SM70-NEXT: shl.b32 %r3, %r2, 3;
+; SM70-NEXT: ld.param.b16 %r4, [or_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r5, %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM70-NEXT: shr.u32 %r7, %r6, %r3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r7;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @or_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: or_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [or_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: or.b32 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB61_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: or_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: or.b32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: or.b32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB62_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: or_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [or_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: or.b32 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB63_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: or_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: or.b32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: or.b32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB64_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: or_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: or.b32 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: or.b32 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB65_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB65_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: or.b32 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: or.b32 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB65_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @or_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: or_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @or_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: or_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @or_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: or_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [or_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @or_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: or_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @or_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: or_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @or_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: or_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [or_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [or_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @xor_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: xor_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM70-NEXT: and.b32 %r2, %r1, 3;
+; SM70-NEXT: shl.b32 %r3, %r2, 3;
+; SM70-NEXT: ld.param.b16 %r4, [xor_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r5, %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM70-NEXT: shr.u32 %r7, %r6, %r3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r7;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @xor_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: xor_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [xor_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: xor.b32 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB73_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: xor_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: xor.b32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: xor.b32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB74_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: xor_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [xor_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: xor.b32 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB75_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: xor_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: xor.b32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: xor.b32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB76_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: xor_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: xor.b32 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: xor.b32 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB77_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB77_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: xor.b32 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: xor.b32 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB77_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @xor_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: xor_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @xor_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: xor_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @xor_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: xor_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xor_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @xor_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: xor_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @xor_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: xor_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @xor_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: xor_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xor_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xor_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: max_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<14>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [max_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [max_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: cvt.s16.s8 %rs7, %rs5;
+; SM70-NEXT: cvt.s16.s8 %rs8, %rs6;
+; SM70-NEXT: $L__BB84_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r16, %r1;
+; SM70-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM70-NEXT: bfe.s32 %r10, %r9, 8, 8;
+; SM70-NEXT: cvt.s8.s32 %rs4, %r10;
+; SM70-NEXT: max.s16 %rs9, %rs4, %rs8;
+; SM70-NEXT: max.s16 %rs10, %rs3, %rs7;
+; SM70-NEXT: and.b16 %rs11, %rs10, 255;
+; SM70-NEXT: shl.b16 %rs12, %rs9, 8;
+; SM70-NEXT: or.b16 %rs13, %rs11, %rs12;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs13;
+; SM70-NEXT: shl.b32 %r12, %r11, %r1;
+; SM70-NEXT: and.b32 %r13, %r16, %r2;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r16;
+; SM70-NEXT: mov.b32 %r16, %r3;
+; SM70-NEXT: @%p1 bra $L__BB84_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: max_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b32 %r<27>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [max_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM70-NEXT: $L__BB85_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
+; SM70-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
+; SM70-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
+; SM70-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
+; SM70-NEXT: setp.gt.s32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r15, %r26, 0, 0x7773U;
+; SM70-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; SM70-NEXT: prmt.b32 %r17, %r26, 0, 0x7772U;
+; SM70-NEXT: selp.b32 %r18, %r17, %r13, %p3;
+; SM70-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
+; SM70-NEXT: prmt.b32 %r20, %r26, 0, 0x7771U;
+; SM70-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; SM70-NEXT: prmt.b32 %r22, %r26, 0, 0x7770U;
+; SM70-NEXT: selp.b32 %r23, %r22, %r11, %p1;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r26;
+; SM70-NEXT: mov.b32 %r26, %r1;
+; SM70-NEXT: @%p5 bra $L__BB85_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: max_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<51>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM70-NEXT: $L__BB86_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r49;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
+; SM70-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
+; SM70-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM70-NEXT: prmt.b32 %r8, %r50, 0, 0xaaa2U;
+; SM70-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM70-NEXT: prmt.b32 %r10, %r50, 0, 0xbbb3U;
+; SM70-NEXT: setp.gt.s32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r1, 0, 0x8880U;
+; SM70-NEXT: prmt.b32 %r12, %r49, 0, 0x8880U;
+; SM70-NEXT: setp.gt.s32 %p5, %r12, %r11;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x9991U;
+; SM70-NEXT: prmt.b32 %r14, %r49, 0, 0x9991U;
+; SM70-NEXT: setp.gt.s32 %p6, %r14, %r13;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0xaaa2U;
+; SM70-NEXT: prmt.b32 %r16, %r49, 0, 0xaaa2U;
+; SM70-NEXT: setp.gt.s32 %p7, %r16, %r15;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0xbbb3U;
+; SM70-NEXT: prmt.b32 %r18, %r49, 0, 0xbbb3U;
+; SM70-NEXT: setp.gt.s32 %p8, %r18, %r17;
+; SM70-NEXT: prmt.b32 %r19, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r19;
+; SM70-NEXT: prmt.b32 %r20, %r50, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r20;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r50, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r22;
+; SM70-NEXT: prmt.b32 %r23, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r23;
+; SM70-NEXT: prmt.b32 %r24, %r50, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r24;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r50, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r26;
+; SM70-NEXT: prmt.b32 %r27, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r27;
+; SM70-NEXT: prmt.b32 %r28, %r49, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r29;
+; SM70-NEXT: prmt.b32 %r30, %r49, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs12, %r30;
+; SM70-NEXT: prmt.b32 %r31, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r31;
+; SM70-NEXT: prmt.b32 %r32, %r49, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r32;
+; SM70-NEXT: prmt.b32 %r33, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r33;
+; SM70-NEXT: prmt.b32 %r34, %r49, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r34;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM70-NEXT: cvt.u32.u16 %r35, %rs17;
+; SM70-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs18;
+; SM70-NEXT: prmt.b32 %r37, %r36, %r35, 0x3340U;
+; SM70-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs19;
+; SM70-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM70-NEXT: cvt.u32.u16 %r39, %rs20;
+; SM70-NEXT: prmt.b32 %r40, %r39, %r38, 0x3340U;
+; SM70-NEXT: prmt.b32 %r41, %r40, %r37, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM70-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r42, %rs21;
+; SM70-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM70-NEXT: cvt.u32.u16 %r43, %rs22;
+; SM70-NEXT: prmt.b32 %r44, %r43, %r42, 0x3340U;
+; SM70-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM70-NEXT: cvt.u32.u16 %r45, %rs23;
+; SM70-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM70-NEXT: cvt.u32.u16 %r46, %rs24;
+; SM70-NEXT: prmt.b32 %r47, %r46, %r45, 0x3340U;
+; SM70-NEXT: prmt.b32 %r48, %r47, %r44, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r48;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB86_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: max_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [max_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB87_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: max.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: max.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB87_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: max_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB88_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: max.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: max.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM70-NEXT: max.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: max.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB88_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: max_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB89_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: max.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: max.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM70-NEXT: max.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: max.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB89_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: $L__BB89_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM70-NEXT: max.s16 %rs17, %rs16, %rs14;
+; SM70-NEXT: max.s16 %rs18, %rs15, %rs13;
+; SM70-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: max.s16 %rs23, %rs22, %rs20;
+; SM70-NEXT: max.s16 %rs24, %rs21, %rs19;
+; SM70-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB89_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @max_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: max_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @max_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: max_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @max_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: max_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [max_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @max_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: max_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @max_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: max_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @max_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: max_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [max_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [max_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: min_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<14>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [min_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [min_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: cvt.s16.s8 %rs7, %rs5;
+; SM70-NEXT: cvt.s16.s8 %rs8, %rs6;
+; SM70-NEXT: $L__BB96_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r16, %r1;
+; SM70-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM70-NEXT: bfe.s32 %r10, %r9, 8, 8;
+; SM70-NEXT: cvt.s8.s32 %rs4, %r10;
+; SM70-NEXT: min.s16 %rs9, %rs4, %rs8;
+; SM70-NEXT: min.s16 %rs10, %rs3, %rs7;
+; SM70-NEXT: and.b16 %rs11, %rs10, 255;
+; SM70-NEXT: shl.b16 %rs12, %rs9, 8;
+; SM70-NEXT: or.b16 %rs13, %rs11, %rs12;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs13;
+; SM70-NEXT: shl.b32 %r12, %r11, %r1;
+; SM70-NEXT: and.b32 %r13, %r16, %r2;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r16;
+; SM70-NEXT: mov.b32 %r16, %r3;
+; SM70-NEXT: @%p1 bra $L__BB96_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: min_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b32 %r<27>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [min_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM70-NEXT: $L__BB97_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
+; SM70-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
+; SM70-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
+; SM70-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
+; SM70-NEXT: setp.le.s32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r15, %r26, 0, 0x7773U;
+; SM70-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; SM70-NEXT: prmt.b32 %r17, %r26, 0, 0x7772U;
+; SM70-NEXT: selp.b32 %r18, %r17, %r13, %p3;
+; SM70-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
+; SM70-NEXT: prmt.b32 %r20, %r26, 0, 0x7771U;
+; SM70-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; SM70-NEXT: prmt.b32 %r22, %r26, 0, 0x7770U;
+; SM70-NEXT: selp.b32 %r23, %r22, %r11, %p1;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r26;
+; SM70-NEXT: mov.b32 %r26, %r1;
+; SM70-NEXT: @%p5 bra $L__BB97_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: min_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<51>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM70-NEXT: $L__BB98_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r49;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
+; SM70-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
+; SM70-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM70-NEXT: prmt.b32 %r8, %r50, 0, 0xaaa2U;
+; SM70-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM70-NEXT: prmt.b32 %r10, %r50, 0, 0xbbb3U;
+; SM70-NEXT: setp.le.s32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r1, 0, 0x8880U;
+; SM70-NEXT: prmt.b32 %r12, %r49, 0, 0x8880U;
+; SM70-NEXT: setp.le.s32 %p5, %r12, %r11;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x9991U;
+; SM70-NEXT: prmt.b32 %r14, %r49, 0, 0x9991U;
+; SM70-NEXT: setp.le.s32 %p6, %r14, %r13;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0xaaa2U;
+; SM70-NEXT: prmt.b32 %r16, %r49, 0, 0xaaa2U;
+; SM70-NEXT: setp.le.s32 %p7, %r16, %r15;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0xbbb3U;
+; SM70-NEXT: prmt.b32 %r18, %r49, 0, 0xbbb3U;
+; SM70-NEXT: setp.le.s32 %p8, %r18, %r17;
+; SM70-NEXT: prmt.b32 %r19, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r19;
+; SM70-NEXT: prmt.b32 %r20, %r50, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r20;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r50, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r22;
+; SM70-NEXT: prmt.b32 %r23, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r23;
+; SM70-NEXT: prmt.b32 %r24, %r50, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r24;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r50, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r26;
+; SM70-NEXT: prmt.b32 %r27, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r27;
+; SM70-NEXT: prmt.b32 %r28, %r49, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r29;
+; SM70-NEXT: prmt.b32 %r30, %r49, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs12, %r30;
+; SM70-NEXT: prmt.b32 %r31, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r31;
+; SM70-NEXT: prmt.b32 %r32, %r49, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r32;
+; SM70-NEXT: prmt.b32 %r33, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r33;
+; SM70-NEXT: prmt.b32 %r34, %r49, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r34;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM70-NEXT: cvt.u32.u16 %r35, %rs17;
+; SM70-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs18;
+; SM70-NEXT: prmt.b32 %r37, %r36, %r35, 0x3340U;
+; SM70-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs19;
+; SM70-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM70-NEXT: cvt.u32.u16 %r39, %rs20;
+; SM70-NEXT: prmt.b32 %r40, %r39, %r38, 0x3340U;
+; SM70-NEXT: prmt.b32 %r41, %r40, %r37, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM70-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r42, %rs21;
+; SM70-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM70-NEXT: cvt.u32.u16 %r43, %rs22;
+; SM70-NEXT: prmt.b32 %r44, %r43, %r42, 0x3340U;
+; SM70-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM70-NEXT: cvt.u32.u16 %r45, %rs23;
+; SM70-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM70-NEXT: cvt.u32.u16 %r46, %rs24;
+; SM70-NEXT: prmt.b32 %r47, %r46, %r45, 0x3340U;
+; SM70-NEXT: prmt.b32 %r48, %r47, %r44, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r48;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB98_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: min_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [min_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB99_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: min.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: min.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB99_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: min_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB100_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: min.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: min.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM70-NEXT: min.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: min.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB100_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: min_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB101_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: min.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: min.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM70-NEXT: min.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: min.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB101_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: $L__BB101_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM70-NEXT: min.s16 %rs17, %rs16, %rs14;
+; SM70-NEXT: min.s16 %rs18, %rs15, %rs13;
+; SM70-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: min.s16 %rs23, %rs22, %rs20;
+; SM70-NEXT: min.s16 %rs24, %rs21, %rs19;
+; SM70-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB101_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @min_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: min_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @min_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: min_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @min_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: min_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [min_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @min_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: min_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @min_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: min_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @min_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: min_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [min_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [min_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: umax_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<18>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umax_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM70-NEXT: $L__BB108_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r17, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM70-NEXT: max.u16 %rs9, %rs5, %rs7;
+; SM70-NEXT: max.u16 %rs10, %rs6, %rs8;
+; SM70-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM70-NEXT: or.b16 %rs12, %rs9, %rs11;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM70-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NEXT: mov.b32 %r17, %r3;
+; SM70-NEXT: @%p1 bra $L__BB108_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: umax_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [umax_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM70-NEXT: $L__BB109_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
+; SM70-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
+; SM70-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
+; SM70-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
+; SM70-NEXT: setp.gt.u32 %p4, %r10, %r9;
+; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p4;
+; SM70-NEXT: selp.b32 %r12, %r8, %r7, %p3;
+; SM70-NEXT: prmt.b32 %r13, %r12, %r11, 0x3340U;
+; SM70-NEXT: selp.b32 %r14, %r6, %r5, %p2;
+; SM70-NEXT: selp.b32 %r15, %r4, %r3, %p1;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r14, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r13, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p5 bra $L__BB109_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: umax_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM70-NEXT: $L__BB110_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM70-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM70-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM70-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM70-NEXT: setp.gt.u32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM70-NEXT: setp.gt.u32 %p5, %r12, %r11;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM70-NEXT: setp.gt.u32 %p6, %r14, %r13;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM70-NEXT: setp.gt.u32 %p7, %r16, %r15;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM70-NEXT: setp.gt.u32 %p8, %r18, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r8;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r9;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r10;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r11;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r12;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r13;
+; SM70-NEXT: cvt.u16.u32 %rs12, %r14;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r15;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r16;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r18;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM70-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM70-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM70-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM70-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM70-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM70-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM70-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM70-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM70-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB110_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: umax_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [umax_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB111_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: max.u16 %rs5, %rs4, %rs2;
+; SM70-NEXT: max.u16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB111_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: umax_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB112_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: max.u16 %rs5, %rs4, %rs2;
+; SM70-NEXT: max.u16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM70-NEXT: max.u16 %rs11, %rs10, %rs8;
+; SM70-NEXT: max.u16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB112_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: umax_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB113_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: max.u16 %rs5, %rs4, %rs2;
+; SM70-NEXT: max.u16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM70-NEXT: max.u16 %rs11, %rs10, %rs8;
+; SM70-NEXT: max.u16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB113_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: $L__BB113_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM70-NEXT: max.u16 %rs17, %rs16, %rs14;
+; SM70-NEXT: max.u16 %rs18, %rs15, %rs13;
+; SM70-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: max.u16 %rs23, %rs22, %rs20;
+; SM70-NEXT: max.u16 %rs24, %rs21, %rs19;
+; SM70-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB113_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @umax_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: umax_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @umax_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: umax_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @umax_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: umax_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umax_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @umax_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: umax_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @umax_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: umax_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @umax_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: umax_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umax_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umax_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: umin_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<18>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umin_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM70-NEXT: $L__BB120_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r17, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM70-NEXT: min.u16 %rs9, %rs5, %rs7;
+; SM70-NEXT: min.u16 %rs10, %rs6, %rs8;
+; SM70-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM70-NEXT: or.b16 %rs12, %rs9, %rs11;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM70-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NEXT: mov.b32 %r17, %r3;
+; SM70-NEXT: @%p1 bra $L__BB120_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: umin_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [umin_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM70-NEXT: $L__BB121_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
+; SM70-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
+; SM70-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
+; SM70-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
+; SM70-NEXT: setp.le.u32 %p4, %r10, %r9;
+; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p4;
+; SM70-NEXT: selp.b32 %r12, %r8, %r7, %p3;
+; SM70-NEXT: prmt.b32 %r13, %r12, %r11, 0x3340U;
+; SM70-NEXT: selp.b32 %r14, %r6, %r5, %p2;
+; SM70-NEXT: selp.b32 %r15, %r4, %r3, %p1;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r14, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r13, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p5 bra $L__BB121_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: umin_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM70-NEXT: $L__BB122_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM70-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM70-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM70-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM70-NEXT: setp.le.u32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM70-NEXT: setp.le.u32 %p5, %r12, %r11;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM70-NEXT: setp.le.u32 %p6, %r14, %r13;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM70-NEXT: setp.le.u32 %p7, %r16, %r15;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM70-NEXT: setp.le.u32 %p8, %r18, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r8;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r9;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r10;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r11;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r12;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r13;
+; SM70-NEXT: cvt.u16.u32 %rs12, %r14;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r15;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r16;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r18;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM70-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM70-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM70-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM70-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM70-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM70-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM70-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM70-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM70-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB122_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: umin_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [umin_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB123_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: min.u16 %rs5, %rs4, %rs2;
+; SM70-NEXT: min.u16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB123_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: umin_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB124_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: min.u16 %rs5, %rs4, %rs2;
+; SM70-NEXT: min.u16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM70-NEXT: min.u16 %rs11, %rs10, %rs8;
+; SM70-NEXT: min.u16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB124_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: umin_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB125_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: min.u16 %rs5, %rs4, %rs2;
+; SM70-NEXT: min.u16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM70-NEXT: min.u16 %rs11, %rs10, %rs8;
+; SM70-NEXT: min.u16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB125_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: $L__BB125_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM70-NEXT: min.u16 %rs17, %rs16, %rs14;
+; SM70-NEXT: min.u16 %rs18, %rs15, %rs13;
+; SM70-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: min.u16 %rs23, %rs22, %rs20;
+; SM70-NEXT: min.u16 %rs24, %rs21, %rs19;
+; SM70-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB125_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @umin_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: umin_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @umin_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: umin_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @umin_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: umin_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umin_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @umin_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: umin_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @umin_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: umin_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @umin_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: umin_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umin_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umin_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<15>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [uinc_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs8, %rs9}, %r4;
+; SM70-NEXT: $L__BB132_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: and.b16 %rs4, %rs3, 255;
+; SM70-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM70-NEXT: add.s16 %rs6, %rs5, 1;
+; SM70-NEXT: add.s16 %rs7, %rs3, 1;
+; SM70-NEXT: setp.ge.u16 %p1, %rs5, %rs9;
+; SM70-NEXT: setp.ge.u16 %p2, %rs4, %rs8;
+; SM70-NEXT: selp.b16 %rs10, 0, %rs7, %p2;
+; SM70-NEXT: selp.b16 %rs11, 0, %rs6, %p1;
+; SM70-NEXT: shl.b16 %rs12, %rs11, 8;
+; SM70-NEXT: and.b16 %rs13, %rs10, 255;
+; SM70-NEXT: or.b16 %rs14, %rs13, %rs12;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs14;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p3 bra $L__BB132_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<23>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r22, [%rd1];
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
+; SM70-NEXT: $L__BB133_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r3, %r22, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: add.s16 %rs2, %rs1, 1;
+; SM70-NEXT: prmt.b32 %r4, %r22, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM70-NEXT: add.s16 %rs4, %rs3, 1;
+; SM70-NEXT: prmt.b32 %r5, %r22, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM70-NEXT: add.s16 %rs6, %rs5, 1;
+; SM70-NEXT: prmt.b32 %r6, %r22, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM70-NEXT: add.s16 %rs8, %rs7, 1;
+; SM70-NEXT: setp.ge.u32 %p1, %r3, %r7;
+; SM70-NEXT: setp.ge.u32 %p2, %r4, %r8;
+; SM70-NEXT: setp.ge.u32 %p3, %r5, %r9;
+; SM70-NEXT: setp.ge.u32 %p4, %r6, %r10;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs8;
+; SM70-NEXT: selp.b32 %r12, 0, %r11, %p4;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs6;
+; SM70-NEXT: selp.b32 %r14, 0, %r13, %p3;
+; SM70-NEXT: prmt.b32 %r15, %r14, %r12, 0x3340U;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs4;
+; SM70-NEXT: selp.b32 %r17, 0, %r16, %p2;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs2;
+; SM70-NEXT: selp.b32 %r19, 0, %r18, %p1;
+; SM70-NEXT: prmt.b32 %r20, %r19, %r17, 0x3340U;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r15, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r22;
+; SM70-NEXT: mov.b32 %r22, %r1;
+; SM70-NEXT: @%p5 bra $L__BB133_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM70-NEXT: $L__BB134_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r3, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: add.s16 %rs2, %rs1, 1;
+; SM70-NEXT: prmt.b32 %r4, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM70-NEXT: add.s16 %rs4, %rs3, 1;
+; SM70-NEXT: prmt.b32 %r5, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM70-NEXT: add.s16 %rs6, %rs5, 1;
+; SM70-NEXT: prmt.b32 %r6, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM70-NEXT: add.s16 %rs8, %rs7, 1;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r7;
+; SM70-NEXT: add.s16 %rs10, %rs9, 1;
+; SM70-NEXT: prmt.b32 %r8, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r8;
+; SM70-NEXT: add.s16 %rs12, %rs11, 1;
+; SM70-NEXT: prmt.b32 %r9, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r9;
+; SM70-NEXT: add.s16 %rs14, %rs13, 1;
+; SM70-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r10;
+; SM70-NEXT: add.s16 %rs16, %rs15, 1;
+; SM70-NEXT: setp.ge.u32 %p1, %r3, %r11;
+; SM70-NEXT: setp.ge.u32 %p2, %r4, %r12;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM70-NEXT: setp.ge.u32 %p3, %r5, %r13;
+; SM70-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM70-NEXT: setp.ge.u32 %p4, %r6, %r14;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0x7770U;
+; SM70-NEXT: setp.ge.u32 %p5, %r7, %r15;
+; SM70-NEXT: prmt.b32 %r16, %r1, 0, 0x7771U;
+; SM70-NEXT: setp.ge.u32 %p6, %r8, %r16;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0x7772U;
+; SM70-NEXT: setp.ge.u32 %p7, %r9, %r17;
+; SM70-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
+; SM70-NEXT: setp.ge.u32 %p8, %r10, %r18;
+; SM70-NEXT: selp.b16 %rs17, 0, %rs16, %p8;
+; SM70-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM70-NEXT: selp.b16 %rs18, 0, %rs14, %p7;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM70-NEXT: selp.b16 %rs19, 0, %rs12, %p6;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM70-NEXT: selp.b16 %rs20, 0, %rs10, %p5;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: selp.b16 %rs21, 0, %rs8, %p4;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM70-NEXT: selp.b16 %rs22, 0, %rs6, %p3;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM70-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM70-NEXT: selp.b16 %rs23, 0, %rs4, %p2;
+; SM70-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM70-NEXT: selp.b16 %rs24, 0, %rs2, %p1;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB134_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: $L__BB135_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs1, 1;
+; SM70-NEXT: add.s16 %rs4, %rs2, 1;
+; SM70-NEXT: setp.ge.u16 %p1, %rs1, %rs5;
+; SM70-NEXT: setp.ge.u16 %p2, %rs2, %rs6;
+; SM70-NEXT: selp.b16 %rs7, 0, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs8, 0, %rs3, %p1;
+; SM70-NEXT: mov.b32 %r3, {%rs8, %rs7};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p3 bra $L__BB135_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM70-NEXT: $L__BB136_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r6;
+; SM70-NEXT: add.s16 %rs3, %rs1, 1;
+; SM70-NEXT: add.s16 %rs4, %rs2, 1;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r5;
+; SM70-NEXT: add.s16 %rs7, %rs5, 1;
+; SM70-NEXT: add.s16 %rs8, %rs6, 1;
+; SM70-NEXT: setp.ge.u16 %p1, %rs1, %rs9;
+; SM70-NEXT: setp.ge.u16 %p2, %rs2, %rs10;
+; SM70-NEXT: setp.ge.u16 %p3, %rs5, %rs11;
+; SM70-NEXT: setp.ge.u16 %p4, %rs6, %rs12;
+; SM70-NEXT: selp.b16 %rs13, 0, %rs8, %p4;
+; SM70-NEXT: selp.b16 %rs14, 0, %rs7, %p3;
+; SM70-NEXT: mov.b32 %r3, {%rs14, %rs13};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: selp.b16 %rs15, 0, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs16, 0, %rs3, %p1;
+; SM70-NEXT: mov.b32 %r4, {%rs16, %rs15};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB136_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<33>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM70-NEXT: $L__BB137_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r10;
+; SM70-NEXT: add.s16 %rs3, %rs1, 1;
+; SM70-NEXT: add.s16 %rs4, %rs2, 1;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r9;
+; SM70-NEXT: add.s16 %rs7, %rs5, 1;
+; SM70-NEXT: add.s16 %rs8, %rs6, 1;
+; SM70-NEXT: setp.ge.u16 %p1, %rs1, %rs9;
+; SM70-NEXT: setp.ge.u16 %p2, %rs2, %rs10;
+; SM70-NEXT: setp.ge.u16 %p3, %rs5, %rs11;
+; SM70-NEXT: setp.ge.u16 %p4, %rs6, %rs12;
+; SM70-NEXT: selp.b16 %rs13, 0, %rs8, %p4;
+; SM70-NEXT: selp.b16 %rs14, 0, %rs7, %p3;
+; SM70-NEXT: mov.b32 %r5, {%rs14, %rs13};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: selp.b16 %rs15, 0, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs16, 0, %rs3, %p1;
+; SM70-NEXT: mov.b32 %r6, {%rs16, %rs15};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB137_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB137_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r12;
+; SM70-NEXT: add.s16 %rs19, %rs17, 1;
+; SM70-NEXT: add.s16 %rs20, %rs18, 1;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r11;
+; SM70-NEXT: add.s16 %rs23, %rs21, 1;
+; SM70-NEXT: add.s16 %rs24, %rs22, 1;
+; SM70-NEXT: mov.b32 {%rs25, %rs26}, %r4;
+; SM70-NEXT: setp.ge.u16 %p6, %rs17, %rs25;
+; SM70-NEXT: setp.ge.u16 %p7, %rs18, %rs26;
+; SM70-NEXT: mov.b32 {%rs27, %rs28}, %r3;
+; SM70-NEXT: setp.ge.u16 %p8, %rs21, %rs27;
+; SM70-NEXT: setp.ge.u16 %p9, %rs22, %rs28;
+; SM70-NEXT: selp.b16 %rs29, 0, %rs24, %p9;
+; SM70-NEXT: selp.b16 %rs30, 0, %rs23, %p8;
+; SM70-NEXT: mov.b32 %r7, {%rs30, %rs29};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: selp.b16 %rs31, 0, %rs20, %p7;
+; SM70-NEXT: selp.b16 %rs32, 0, %rs19, %p6;
+; SM70-NEXT: mov.b32 %r8, {%rs32, %rs31};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB137_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @uinc_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @uinc_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @uinc_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [uinc_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
+; SM70-NEXT: $L__BB141_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd6, %rd10, 1;
+; SM70-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
+; SM70-NEXT: selp.b64 %rd7, 0, %rd6, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM70-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
+; SM70-NEXT: mov.b64 %rd10, %rd3;
+; SM70-NEXT: @%p2 bra $L__BB141_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
+; SM70-NEXT: $L__BB141_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd8, %rd11, 1;
+; SM70-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
+; SM70-NEXT: selp.b64 %rd9, 0, %rd8, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM70-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
+; SM70-NEXT: mov.b64 %rd11, %rd4;
+; SM70-NEXT: @%p4 bra $L__BB141_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [uinc_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
+; SM70-NEXT: $L__BB142_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd18;
+; SM70-NEXT: add.s64 %rd10, %rd5, 1;
+; SM70-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
+; SM70-NEXT: selp.b64 %rd11, 0, %rd10, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
+; SM70-NEXT: @%p2 bra $L__BB142_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
+; SM70-NEXT: $L__BB142_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd19;
+; SM70-NEXT: add.s64 %rd12, %rd6, 1;
+; SM70-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
+; SM70-NEXT: selp.b64 %rd13, 0, %rd12, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
+; SM70-NEXT: @%p4 bra $L__BB142_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
+; SM70-NEXT: $L__BB142_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd14, %rd20, 1;
+; SM70-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
+; SM70-NEXT: selp.b64 %rd15, 0, %rd14, %p5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM70-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
+; SM70-NEXT: mov.b64 %rd20, %rd7;
+; SM70-NEXT: @%p6 bra $L__BB142_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
+; SM70-NEXT: $L__BB142_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd16, %rd21, 1;
+; SM70-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
+; SM70-NEXT: selp.b64 %rd17, 0, %rd16, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM70-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
+; SM70-NEXT: mov.b64 %rd21, %rd8;
+; SM70-NEXT: @%p8 bra $L__BB142_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<17>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [uinc_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
+; SM70-NEXT: $L__BB143_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd34;
+; SM70-NEXT: add.s64 %rd18, %rd7, 1;
+; SM70-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
+; SM70-NEXT: selp.b64 %rd19, 0, %rd18, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
+; SM70-NEXT: @%p2 bra $L__BB143_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
+; SM70-NEXT: $L__BB143_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd35;
+; SM70-NEXT: add.s64 %rd20, %rd8, 1;
+; SM70-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
+; SM70-NEXT: selp.b64 %rd21, 0, %rd20, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB143_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
+; SM70-NEXT: $L__BB143_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd36;
+; SM70-NEXT: add.s64 %rd22, %rd9, 1;
+; SM70-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
+; SM70-NEXT: selp.b64 %rd23, 0, %rd22, %p5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
+; SM70-NEXT: @%p6 bra $L__BB143_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
+; SM70-NEXT: $L__BB143_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd37;
+; SM70-NEXT: add.s64 %rd24, %rd10, 1;
+; SM70-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
+; SM70-NEXT: selp.b64 %rd25, 0, %rd24, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
+; SM70-NEXT: @%p8 bra $L__BB143_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
+; SM70-NEXT: $L__BB143_9: // %atomicrmw.start41
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd38;
+; SM70-NEXT: add.s64 %rd26, %rd13, 1;
+; SM70-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
+; SM70-NEXT: selp.b64 %rd27, 0, %rd26, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM70-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
+; SM70-NEXT: @%p10 bra $L__BB143_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end40
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
+; SM70-NEXT: $L__BB143_11: // %atomicrmw.start47
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd39;
+; SM70-NEXT: add.s64 %rd28, %rd14, 1;
+; SM70-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
+; SM70-NEXT: selp.b64 %rd29, 0, %rd28, %p11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM70-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
+; SM70-NEXT: @%p12 bra $L__BB143_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end46
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
+; SM70-NEXT: $L__BB143_13: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd30, %rd40, 1;
+; SM70-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
+; SM70-NEXT: selp.b64 %rd31, 0, %rd30, %p13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM70-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
+; SM70-NEXT: mov.b64 %rd40, %rd15;
+; SM70-NEXT: @%p14 bra $L__BB143_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end57
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
+; SM70-NEXT: $L__BB143_15: // %atomicrmw.start64
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd32, %rd41, 1;
+; SM70-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
+; SM70-NEXT: selp.b64 %rd33, 0, %rd32, %p15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM70-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
+; SM70-NEXT: mov.b64 %rd41, %rd16;
+; SM70-NEXT: @%p16 bra $L__BB143_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end63
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [udec_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs8, %rs9}, %r4;
+; SM70-NEXT: $L__BB144_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: and.b16 %rs4, %rs3, 255;
+; SM70-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM70-NEXT: add.s16 %rs6, %rs5, -1;
+; SM70-NEXT: add.s16 %rs7, %rs3, -1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs5, 0;
+; SM70-NEXT: setp.eq.b16 %p2, %rs4, 0;
+; SM70-NEXT: setp.gt.u16 %p3, %rs5, %rs9;
+; SM70-NEXT: setp.gt.u16 %p4, %rs4, %rs8;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs7, %p4;
+; SM70-NEXT: selp.b16 %rs11, %rs8, %rs10, %p2;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs13, %rs9, %rs12, %p1;
+; SM70-NEXT: shl.b16 %rs14, %rs13, 8;
+; SM70-NEXT: and.b16 %rs15, %rs11, 255;
+; SM70-NEXT: or.b16 %rs16, %rs15, %rs14;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs16;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p5, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p5 bra $L__BB144_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<27>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
+; SM70-NEXT: $L__BB145_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r3, %r26, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: add.s16 %rs2, %rs1, -1;
+; SM70-NEXT: prmt.b32 %r4, %r26, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM70-NEXT: add.s16 %rs4, %rs3, -1;
+; SM70-NEXT: prmt.b32 %r5, %r26, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM70-NEXT: add.s16 %rs6, %rs5, -1;
+; SM70-NEXT: prmt.b32 %r6, %r26, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM70-NEXT: add.s16 %rs8, %rs7, -1;
+; SM70-NEXT: setp.eq.b32 %p1, %r3, 0;
+; SM70-NEXT: setp.eq.b32 %p2, %r4, 0;
+; SM70-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM70-NEXT: setp.eq.b32 %p4, %r6, 0;
+; SM70-NEXT: setp.gt.u32 %p5, %r3, %r7;
+; SM70-NEXT: setp.gt.u32 %p6, %r4, %r8;
+; SM70-NEXT: setp.gt.u32 %p7, %r5, %r9;
+; SM70-NEXT: setp.gt.u32 %p8, %r6, %r10;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs8;
+; SM70-NEXT: selp.b32 %r12, %r10, %r11, %p8;
+; SM70-NEXT: selp.b32 %r13, %r10, %r12, %p4;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs6;
+; SM70-NEXT: selp.b32 %r15, %r9, %r14, %p7;
+; SM70-NEXT: selp.b32 %r16, %r9, %r15, %p3;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r13, 0x3340U;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs4;
+; SM70-NEXT: selp.b32 %r19, %r8, %r18, %p6;
+; SM70-NEXT: selp.b32 %r20, %r8, %r19, %p2;
+; SM70-NEXT: cvt.u32.u16 %r21, %rs2;
+; SM70-NEXT: selp.b32 %r22, %r7, %r21, %p5;
+; SM70-NEXT: selp.b32 %r23, %r7, %r22, %p1;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r17, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM70-NEXT: setp.ne.b32 %p9, %r1, %r26;
+; SM70-NEXT: mov.b32 %r26, %r1;
+; SM70-NEXT: @%p9 bra $L__BB145_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<18>;
+; SM70-NEXT: .reg .b16 %rs<41>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r11;
+; SM70-NEXT: cvt.u16.u32 %rs18, %r12;
+; SM70-NEXT: $L__BB146_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r3, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: add.s16 %rs2, %rs1, -1;
+; SM70-NEXT: prmt.b32 %r4, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM70-NEXT: add.s16 %rs4, %rs3, -1;
+; SM70-NEXT: prmt.b32 %r5, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM70-NEXT: add.s16 %rs6, %rs5, -1;
+; SM70-NEXT: prmt.b32 %r6, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM70-NEXT: add.s16 %rs8, %rs7, -1;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r7;
+; SM70-NEXT: add.s16 %rs10, %rs9, -1;
+; SM70-NEXT: prmt.b32 %r8, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r8;
+; SM70-NEXT: add.s16 %rs12, %rs11, -1;
+; SM70-NEXT: prmt.b32 %r9, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r9;
+; SM70-NEXT: add.s16 %rs14, %rs13, -1;
+; SM70-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r10;
+; SM70-NEXT: add.s16 %rs16, %rs15, -1;
+; SM70-NEXT: setp.eq.b32 %p1, %r3, 0;
+; SM70-NEXT: setp.eq.b32 %p2, %r4, 0;
+; SM70-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM70-NEXT: setp.eq.b32 %p4, %r6, 0;
+; SM70-NEXT: setp.eq.b32 %p5, %r7, 0;
+; SM70-NEXT: setp.eq.b32 %p6, %r8, 0;
+; SM70-NEXT: setp.eq.b32 %p7, %r9, 0;
+; SM70-NEXT: setp.eq.b32 %p8, %r10, 0;
+; SM70-NEXT: setp.gt.u32 %p9, %r3, %r11;
+; SM70-NEXT: setp.gt.u32 %p10, %r4, %r12;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM70-NEXT: setp.gt.u32 %p11, %r5, %r13;
+; SM70-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM70-NEXT: setp.gt.u32 %p12, %r6, %r14;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0x7770U;
+; SM70-NEXT: setp.gt.u32 %p13, %r7, %r15;
+; SM70-NEXT: prmt.b32 %r16, %r1, 0, 0x7771U;
+; SM70-NEXT: setp.gt.u32 %p14, %r8, %r16;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0x7772U;
+; SM70-NEXT: setp.gt.u32 %p15, %r9, %r17;
+; SM70-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
+; SM70-NEXT: setp.gt.u32 %p16, %r10, %r18;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r13;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r14;
+; SM70-NEXT: cvt.u16.u32 %rs21, %r15;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r16;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs24, %r18;
+; SM70-NEXT: selp.b16 %rs25, %rs24, %rs16, %p16;
+; SM70-NEXT: selp.b16 %rs26, %rs24, %rs25, %p8;
+; SM70-NEXT: cvt.u32.u16 %r19, %rs26;
+; SM70-NEXT: selp.b16 %rs27, %rs23, %rs14, %p15;
+; SM70-NEXT: selp.b16 %rs28, %rs23, %rs27, %p7;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs28;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM70-NEXT: selp.b16 %rs29, %rs22, %rs12, %p14;
+; SM70-NEXT: selp.b16 %rs30, %rs22, %rs29, %p6;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs30;
+; SM70-NEXT: selp.b16 %rs31, %rs21, %rs10, %p13;
+; SM70-NEXT: selp.b16 %rs32, %rs21, %rs31, %p5;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs32;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: selp.b16 %rs33, %rs20, %rs8, %p12;
+; SM70-NEXT: selp.b16 %rs34, %rs20, %rs33, %p4;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs34;
+; SM70-NEXT: selp.b16 %rs35, %rs19, %rs6, %p11;
+; SM70-NEXT: selp.b16 %rs36, %rs19, %rs35, %p3;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs36;
+; SM70-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM70-NEXT: selp.b16 %rs37, %rs18, %rs4, %p10;
+; SM70-NEXT: selp.b16 %rs38, %rs18, %rs37, %p2;
+; SM70-NEXT: cvt.u32.u16 %r29, %rs38;
+; SM70-NEXT: selp.b16 %rs39, %rs17, %rs2, %p9;
+; SM70-NEXT: selp.b16 %rs40, %rs17, %rs39, %p1;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs40;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p17, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p17 bra $L__BB146_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<11>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: $L__BB147_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs1, -1;
+; SM70-NEXT: add.s16 %rs4, %rs2, -1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM70-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM70-NEXT: setp.gt.u16 %p3, %rs1, %rs5;
+; SM70-NEXT: setp.gt.u16 %p4, %rs2, %rs6;
+; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p4;
+; SM70-NEXT: selp.b16 %rs8, %rs6, %rs7, %p2;
+; SM70-NEXT: selp.b16 %rs9, %rs5, %rs3, %p3;
+; SM70-NEXT: selp.b16 %rs10, %rs5, %rs9, %p1;
+; SM70-NEXT: mov.b32 %r3, {%rs10, %rs8};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p5 bra $L__BB147_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<21>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM70-NEXT: $L__BB148_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r6;
+; SM70-NEXT: add.s16 %rs3, %rs1, -1;
+; SM70-NEXT: add.s16 %rs4, %rs2, -1;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r5;
+; SM70-NEXT: add.s16 %rs7, %rs5, -1;
+; SM70-NEXT: add.s16 %rs8, %rs6, -1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM70-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM70-NEXT: setp.eq.b16 %p3, %rs5, 0;
+; SM70-NEXT: setp.eq.b16 %p4, %rs6, 0;
+; SM70-NEXT: setp.gt.u16 %p5, %rs1, %rs9;
+; SM70-NEXT: setp.gt.u16 %p6, %rs2, %rs10;
+; SM70-NEXT: setp.gt.u16 %p7, %rs5, %rs11;
+; SM70-NEXT: setp.gt.u16 %p8, %rs6, %rs12;
+; SM70-NEXT: selp.b16 %rs13, %rs12, %rs8, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs12, %rs13, %p4;
+; SM70-NEXT: selp.b16 %rs15, %rs11, %rs7, %p7;
+; SM70-NEXT: selp.b16 %rs16, %rs11, %rs15, %p3;
+; SM70-NEXT: mov.b32 %r3, {%rs16, %rs14};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: selp.b16 %rs17, %rs10, %rs4, %p6;
+; SM70-NEXT: selp.b16 %rs18, %rs10, %rs17, %p2;
+; SM70-NEXT: selp.b16 %rs19, %rs9, %rs3, %p5;
+; SM70-NEXT: selp.b16 %rs20, %rs9, %rs19, %p1;
+; SM70-NEXT: mov.b32 %r4, {%rs20, %rs18};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB148_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<19>;
+; SM70-NEXT: .reg .b16 %rs<41>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM70-NEXT: $L__BB149_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r10;
+; SM70-NEXT: add.s16 %rs3, %rs1, -1;
+; SM70-NEXT: add.s16 %rs4, %rs2, -1;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r9;
+; SM70-NEXT: add.s16 %rs7, %rs5, -1;
+; SM70-NEXT: add.s16 %rs8, %rs6, -1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM70-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM70-NEXT: setp.eq.b16 %p3, %rs5, 0;
+; SM70-NEXT: setp.eq.b16 %p4, %rs6, 0;
+; SM70-NEXT: setp.gt.u16 %p5, %rs1, %rs9;
+; SM70-NEXT: setp.gt.u16 %p6, %rs2, %rs10;
+; SM70-NEXT: setp.gt.u16 %p7, %rs5, %rs11;
+; SM70-NEXT: setp.gt.u16 %p8, %rs6, %rs12;
+; SM70-NEXT: selp.b16 %rs13, %rs12, %rs8, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs12, %rs13, %p4;
+; SM70-NEXT: selp.b16 %rs15, %rs11, %rs7, %p7;
+; SM70-NEXT: selp.b16 %rs16, %rs11, %rs15, %p3;
+; SM70-NEXT: mov.b32 %r5, {%rs16, %rs14};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: selp.b16 %rs17, %rs10, %rs4, %p6;
+; SM70-NEXT: selp.b16 %rs18, %rs10, %rs17, %p2;
+; SM70-NEXT: selp.b16 %rs19, %rs9, %rs3, %p5;
+; SM70-NEXT: selp.b16 %rs20, %rs9, %rs19, %p1;
+; SM70-NEXT: mov.b32 %r6, {%rs20, %rs18};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB149_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB149_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: add.s16 %rs23, %rs21, -1;
+; SM70-NEXT: add.s16 %rs24, %rs22, -1;
+; SM70-NEXT: mov.b32 {%rs25, %rs26}, %r11;
+; SM70-NEXT: add.s16 %rs27, %rs25, -1;
+; SM70-NEXT: add.s16 %rs28, %rs26, -1;
+; SM70-NEXT: setp.eq.b16 %p10, %rs21, 0;
+; SM70-NEXT: setp.eq.b16 %p11, %rs22, 0;
+; SM70-NEXT: setp.eq.b16 %p12, %rs25, 0;
+; SM70-NEXT: setp.eq.b16 %p13, %rs26, 0;
+; SM70-NEXT: mov.b32 {%rs29, %rs30}, %r4;
+; SM70-NEXT: setp.gt.u16 %p14, %rs21, %rs29;
+; SM70-NEXT: setp.gt.u16 %p15, %rs22, %rs30;
+; SM70-NEXT: mov.b32 {%rs31, %rs32}, %r3;
+; SM70-NEXT: setp.gt.u16 %p16, %rs25, %rs31;
+; SM70-NEXT: setp.gt.u16 %p17, %rs26, %rs32;
+; SM70-NEXT: selp.b16 %rs33, %rs32, %rs28, %p17;
+; SM70-NEXT: selp.b16 %rs34, %rs32, %rs33, %p13;
+; SM70-NEXT: selp.b16 %rs35, %rs31, %rs27, %p16;
+; SM70-NEXT: selp.b16 %rs36, %rs31, %rs35, %p12;
+; SM70-NEXT: mov.b32 %r7, {%rs36, %rs34};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: selp.b16 %rs37, %rs30, %rs24, %p15;
+; SM70-NEXT: selp.b16 %rs38, %rs30, %rs37, %p11;
+; SM70-NEXT: selp.b16 %rs39, %rs29, %rs23, %p14;
+; SM70-NEXT: selp.b16 %rs40, %rs29, %rs39, %p10;
+; SM70-NEXT: mov.b32 %r8, {%rs40, %rs38};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p18 bra $L__BB149_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @udec_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @udec_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @udec_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [udec_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<7>;
+; SM70-NEXT: .reg .b64 %rd<14>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [udec_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd5];
+; SM70-NEXT: $L__BB153_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd6, %rd12, -1;
+; SM70-NEXT: setp.eq.b64 %p1, %rd12, 0;
+; SM70-NEXT: setp.gt.u64 %p2, %rd12, %rd1;
+; SM70-NEXT: selp.b64 %rd7, %rd1, %rd6, %p2;
+; SM70-NEXT: selp.b64 %rd8, %rd1, %rd7, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd12, %rd8;
+; SM70-NEXT: setp.ne.b64 %p3, %rd3, %rd12;
+; SM70-NEXT: mov.b64 %rd12, %rd3;
+; SM70-NEXT: @%p3 bra $L__BB153_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd5+8];
+; SM70-NEXT: $L__BB153_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd9, %rd13, -1;
+; SM70-NEXT: setp.eq.b64 %p4, %rd13, 0;
+; SM70-NEXT: setp.gt.u64 %p5, %rd13, %rd2;
+; SM70-NEXT: selp.b64 %rd10, %rd2, %rd9, %p5;
+; SM70-NEXT: selp.b64 %rd11, %rd2, %rd10, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd13, %rd11;
+; SM70-NEXT: setp.ne.b64 %p6, %rd4, %rd13;
+; SM70-NEXT: mov.b64 %rd13, %rd4;
+; SM70-NEXT: @%p6 bra $L__BB153_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<13>;
+; SM70-NEXT: .reg .b64 %rd<26>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [udec_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd9];
+; SM70-NEXT: $L__BB154_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd22;
+; SM70-NEXT: add.s64 %rd10, %rd5, -1;
+; SM70-NEXT: setp.eq.b64 %p1, %rd5, 0;
+; SM70-NEXT: setp.gt.u64 %p2, %rd5, %rd3;
+; SM70-NEXT: selp.b64 %rd11, %rd3, %rd10, %p2;
+; SM70-NEXT: selp.b64 %rd12, %rd3, %rd11, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd22, [%rd9], %rd5, %rd12;
+; SM70-NEXT: setp.ne.b64 %p3, %rd22, %rd5;
+; SM70-NEXT: @%p3 bra $L__BB154_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd23, [%rd9+8];
+; SM70-NEXT: $L__BB154_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd23;
+; SM70-NEXT: add.s64 %rd13, %rd6, -1;
+; SM70-NEXT: setp.eq.b64 %p4, %rd6, 0;
+; SM70-NEXT: setp.gt.u64 %p5, %rd6, %rd4;
+; SM70-NEXT: selp.b64 %rd14, %rd4, %rd13, %p5;
+; SM70-NEXT: selp.b64 %rd15, %rd4, %rd14, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd9+8], %rd6, %rd15;
+; SM70-NEXT: setp.ne.b64 %p6, %rd23, %rd6;
+; SM70-NEXT: @%p6 bra $L__BB154_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd9+16];
+; SM70-NEXT: $L__BB154_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd16, %rd24, -1;
+; SM70-NEXT: setp.eq.b64 %p7, %rd24, 0;
+; SM70-NEXT: setp.gt.u64 %p8, %rd24, %rd1;
+; SM70-NEXT: selp.b64 %rd17, %rd1, %rd16, %p8;
+; SM70-NEXT: selp.b64 %rd18, %rd1, %rd17, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd24, %rd18;
+; SM70-NEXT: setp.ne.b64 %p9, %rd7, %rd24;
+; SM70-NEXT: mov.b64 %rd24, %rd7;
+; SM70-NEXT: @%p9 bra $L__BB154_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd25, [%rd9+24];
+; SM70-NEXT: $L__BB154_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd19, %rd25, -1;
+; SM70-NEXT: setp.eq.b64 %p10, %rd25, 0;
+; SM70-NEXT: setp.gt.u64 %p11, %rd25, %rd2;
+; SM70-NEXT: selp.b64 %rd20, %rd2, %rd19, %p11;
+; SM70-NEXT: selp.b64 %rd21, %rd2, %rd20, %p10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd25, %rd21;
+; SM70-NEXT: setp.ne.b64 %p12, %rd8, %rd25;
+; SM70-NEXT: mov.b64 %rd25, %rd8;
+; SM70-NEXT: @%p12 bra $L__BB154_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd22, %rd23};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<25>;
+; SM70-NEXT: .reg .b64 %rd<50>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [udec_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd42, [%rd17];
+; SM70-NEXT: $L__BB155_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd42;
+; SM70-NEXT: add.s64 %rd18, %rd7, -1;
+; SM70-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM70-NEXT: setp.gt.u64 %p2, %rd7, %rd5;
+; SM70-NEXT: selp.b64 %rd19, %rd5, %rd18, %p2;
+; SM70-NEXT: selp.b64 %rd20, %rd5, %rd19, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd42, [%rd17], %rd7, %rd20;
+; SM70-NEXT: setp.ne.b64 %p3, %rd42, %rd7;
+; SM70-NEXT: @%p3 bra $L__BB155_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd43, [%rd17+8];
+; SM70-NEXT: $L__BB155_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd43;
+; SM70-NEXT: add.s64 %rd21, %rd8, -1;
+; SM70-NEXT: setp.eq.b64 %p4, %rd8, 0;
+; SM70-NEXT: setp.gt.u64 %p5, %rd8, %rd6;
+; SM70-NEXT: selp.b64 %rd22, %rd6, %rd21, %p5;
+; SM70-NEXT: selp.b64 %rd23, %rd6, %rd22, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd43, [%rd17+8], %rd8, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd43, %rd8;
+; SM70-NEXT: @%p6 bra $L__BB155_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd44, [%rd17+16];
+; SM70-NEXT: $L__BB155_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd44;
+; SM70-NEXT: add.s64 %rd24, %rd9, -1;
+; SM70-NEXT: setp.eq.b64 %p7, %rd9, 0;
+; SM70-NEXT: setp.gt.u64 %p8, %rd9, %rd3;
+; SM70-NEXT: selp.b64 %rd25, %rd3, %rd24, %p8;
+; SM70-NEXT: selp.b64 %rd26, %rd3, %rd25, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd44, [%rd17+16], %rd9, %rd26;
+; SM70-NEXT: setp.ne.b64 %p9, %rd44, %rd9;
+; SM70-NEXT: @%p9 bra $L__BB155_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd45, [%rd17+24];
+; SM70-NEXT: $L__BB155_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd45;
+; SM70-NEXT: add.s64 %rd27, %rd10, -1;
+; SM70-NEXT: setp.eq.b64 %p10, %rd10, 0;
+; SM70-NEXT: setp.gt.u64 %p11, %rd10, %rd4;
+; SM70-NEXT: selp.b64 %rd28, %rd4, %rd27, %p11;
+; SM70-NEXT: selp.b64 %rd29, %rd4, %rd28, %p10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd17+24], %rd10, %rd29;
+; SM70-NEXT: setp.ne.b64 %p12, %rd45, %rd10;
+; SM70-NEXT: @%p12 bra $L__BB155_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd46, [%rd17+32];
+; SM70-NEXT: $L__BB155_9: // %atomicrmw.start41
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd46;
+; SM70-NEXT: add.s64 %rd30, %rd13, -1;
+; SM70-NEXT: setp.eq.b64 %p13, %rd13, 0;
+; SM70-NEXT: setp.gt.u64 %p14, %rd13, %rd1;
+; SM70-NEXT: selp.b64 %rd31, %rd1, %rd30, %p14;
+; SM70-NEXT: selp.b64 %rd32, %rd1, %rd31, %p13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd46, [%rd17+32], %rd13, %rd32;
+; SM70-NEXT: setp.ne.b64 %p15, %rd46, %rd13;
+; SM70-NEXT: @%p15 bra $L__BB155_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end40
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd47, [%rd17+40];
+; SM70-NEXT: $L__BB155_11: // %atomicrmw.start47
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd47;
+; SM70-NEXT: add.s64 %rd33, %rd14, -1;
+; SM70-NEXT: setp.eq.b64 %p16, %rd14, 0;
+; SM70-NEXT: setp.gt.u64 %p17, %rd14, %rd2;
+; SM70-NEXT: selp.b64 %rd34, %rd2, %rd33, %p17;
+; SM70-NEXT: selp.b64 %rd35, %rd2, %rd34, %p16;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd47, [%rd17+40], %rd14, %rd35;
+; SM70-NEXT: setp.ne.b64 %p18, %rd47, %rd14;
+; SM70-NEXT: @%p18 bra $L__BB155_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end46
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd48, [%rd17+48];
+; SM70-NEXT: $L__BB155_13: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd36, %rd48, -1;
+; SM70-NEXT: setp.eq.b64 %p19, %rd48, 0;
+; SM70-NEXT: setp.gt.u64 %p20, %rd48, %rd11;
+; SM70-NEXT: selp.b64 %rd37, %rd11, %rd36, %p20;
+; SM70-NEXT: selp.b64 %rd38, %rd11, %rd37, %p19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd48, %rd38;
+; SM70-NEXT: setp.ne.b64 %p21, %rd15, %rd48;
+; SM70-NEXT: mov.b64 %rd48, %rd15;
+; SM70-NEXT: @%p21 bra $L__BB155_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end57
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd49, [%rd17+56];
+; SM70-NEXT: $L__BB155_15: // %atomicrmw.start64
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd39, %rd49, -1;
+; SM70-NEXT: setp.eq.b64 %p22, %rd49, 0;
+; SM70-NEXT: setp.gt.u64 %p23, %rd49, %rd12;
+; SM70-NEXT: selp.b64 %rd40, %rd12, %rd39, %p23;
+; SM70-NEXT: selp.b64 %rd41, %rd12, %rd40, %p22;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd49, %rd41;
+; SM70-NEXT: setp.ne.b64 %p24, %rd16, %rd49;
+; SM70-NEXT: mov.b64 %rd49, %rd16;
+; SM70-NEXT: @%p24 bra $L__BB155_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end63
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd46, %rd47};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<15>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_cond_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs6, %rs7}, %r4;
+; SM70-NEXT: $L__BB156_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: and.b16 %rs4, %rs3, 255;
+; SM70-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM70-NEXT: setp.ge.u16 %p1, %rs5, %rs7;
+; SM70-NEXT: setp.ge.u16 %p2, %rs4, %rs6;
+; SM70-NEXT: sub.s16 %rs8, %rs5, %rs7;
+; SM70-NEXT: sub.s16 %rs9, %rs3, %rs6;
+; SM70-NEXT: selp.b16 %rs10, %rs9, %rs3, %p2;
+; SM70-NEXT: selp.b16 %rs11, %rs8, %rs5, %p1;
+; SM70-NEXT: shl.b16 %rs12, %rs11, 8;
+; SM70-NEXT: and.b16 %rs13, %rs10, 255;
+; SM70-NEXT: or.b16 %rs14, %rs13, %rs12;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs14;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p3 bra $L__BB156_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<23>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r22, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r9;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r3;
+; SM70-NEXT: $L__BB157_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r22, 0, 0x7770U;
+; SM70-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r22, 0, 0x7771U;
+; SM70-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r8, %r22, 0, 0x7772U;
+; SM70-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r10, %r22, 0, 0x7773U;
+; SM70-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r4;
+; SM70-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM70-NEXT: sub.s16 %rs8, %rs7, %rs3;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r8;
+; SM70-NEXT: sub.s16 %rs10, %rs9, %rs2;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r10;
+; SM70-NEXT: sub.s16 %rs12, %rs11, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs12;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p4;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs10;
+; SM70-NEXT: selp.b32 %r14, %r13, %r8, %p3;
+; SM70-NEXT: prmt.b32 %r15, %r14, %r12, 0x3340U;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs8;
+; SM70-NEXT: selp.b32 %r17, %r16, %r6, %p2;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs6;
+; SM70-NEXT: selp.b32 %r19, %r18, %r4, %p1;
+; SM70-NEXT: prmt.b32 %r20, %r19, %r17, 0x3340U;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r15, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r22;
+; SM70-NEXT: mov.b32 %r22, %r1;
+; SM70-NEXT: @%p5 bra $L__BB157_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<33>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r5;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r3;
+; SM70-NEXT: $L__BB158_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM70-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM70-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM70-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM70-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM70-NEXT: setp.ge.u32 %p5, %r12, %r11;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM70-NEXT: setp.ge.u32 %p6, %r14, %r13;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM70-NEXT: setp.ge.u32 %p7, %r16, %r15;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM70-NEXT: setp.ge.u32 %p8, %r18, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r18;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r15;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r16;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r13;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r14;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r11;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r12;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r9;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r10;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r7;
+; SM70-NEXT: cvt.u16.u32 %rs12, %r8;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r6;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r4;
+; SM70-NEXT: sub.s16 %rs17, %rs16, %rs15;
+; SM70-NEXT: sub.s16 %rs18, %rs14, %rs13;
+; SM70-NEXT: sub.s16 %rs19, %rs12, %rs11;
+; SM70-NEXT: sub.s16 %rs20, %rs10, %rs9;
+; SM70-NEXT: sub.s16 %rs21, %rs8, %rs7;
+; SM70-NEXT: sub.s16 %rs22, %rs6, %rs5;
+; SM70-NEXT: sub.s16 %rs23, %rs4, %rs3;
+; SM70-NEXT: sub.s16 %rs24, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs25, %rs24, %rs2, %p8;
+; SM70-NEXT: cvt.u32.u16 %r19, %rs25;
+; SM70-NEXT: selp.b16 %rs26, %rs23, %rs4, %p7;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs26;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM70-NEXT: selp.b16 %rs27, %rs22, %rs6, %p6;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs27;
+; SM70-NEXT: selp.b16 %rs28, %rs21, %rs8, %p5;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs28;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: selp.b16 %rs29, %rs20, %rs10, %p4;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs29;
+; SM70-NEXT: selp.b16 %rs30, %rs19, %rs12, %p3;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs30;
+; SM70-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM70-NEXT: selp.b16 %rs31, %rs18, %rs14, %p2;
+; SM70-NEXT: cvt.u32.u16 %r29, %rs31;
+; SM70-NEXT: selp.b16 %rs32, %rs17, %rs16, %p1;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs32;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB158_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB159_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM70-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs5, %rs3, %rs1;
+; SM70-NEXT: sub.s16 %rs6, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs3, %p1;
+; SM70-NEXT: mov.b32 %r3, {%rs8, %rs7};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p3 bra $L__BB159_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; SM70-NEXT: $L__BB160_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r6;
+; SM70-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM70-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r5;
+; SM70-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM70-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM70-NEXT: sub.s16 %rs9, %rs3, %rs1;
+; SM70-NEXT: sub.s16 %rs10, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs11, %rs7, %rs5;
+; SM70-NEXT: sub.s16 %rs12, %rs8, %rs6;
+; SM70-NEXT: selp.b16 %rs13, %rs12, %rs8, %p4;
+; SM70-NEXT: selp.b16 %rs14, %rs11, %rs7, %p3;
+; SM70-NEXT: mov.b32 %r3, {%rs14, %rs13};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: selp.b16 %rs15, %rs10, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs16, %rs9, %rs3, %p1;
+; SM70-NEXT: mov.b32 %r4, {%rs16, %rs15};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB160_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<33>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; SM70-NEXT: $L__BB161_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r10;
+; SM70-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM70-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r9;
+; SM70-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM70-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM70-NEXT: sub.s16 %rs9, %rs3, %rs1;
+; SM70-NEXT: sub.s16 %rs10, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs11, %rs7, %rs5;
+; SM70-NEXT: sub.s16 %rs12, %rs8, %rs6;
+; SM70-NEXT: selp.b16 %rs13, %rs12, %rs8, %p4;
+; SM70-NEXT: selp.b16 %rs14, %rs11, %rs7, %p3;
+; SM70-NEXT: mov.b32 %r5, {%rs14, %rs13};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: selp.b16 %rs15, %rs10, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs16, %rs9, %rs3, %p1;
+; SM70-NEXT: mov.b32 %r6, {%rs16, %rs15};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB161_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB161_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r4;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r12;
+; SM70-NEXT: setp.ge.u16 %p6, %rs19, %rs17;
+; SM70-NEXT: setp.ge.u16 %p7, %rs20, %rs18;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r3;
+; SM70-NEXT: mov.b32 {%rs23, %rs24}, %r11;
+; SM70-NEXT: setp.ge.u16 %p8, %rs23, %rs21;
+; SM70-NEXT: setp.ge.u16 %p9, %rs24, %rs22;
+; SM70-NEXT: sub.s16 %rs25, %rs19, %rs17;
+; SM70-NEXT: sub.s16 %rs26, %rs20, %rs18;
+; SM70-NEXT: sub.s16 %rs27, %rs23, %rs21;
+; SM70-NEXT: sub.s16 %rs28, %rs24, %rs22;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs24, %p9;
+; SM70-NEXT: selp.b16 %rs30, %rs27, %rs23, %p8;
+; SM70-NEXT: mov.b32 %r7, {%rs30, %rs29};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: selp.b16 %rs31, %rs26, %rs20, %p7;
+; SM70-NEXT: selp.b16 %rs32, %rs25, %rs19, %p6;
+; SM70-NEXT: mov.b32 %r8, {%rs32, %rs31};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB161_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB162_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: setp.ge.u32 %p1, %r8, %r2;
+; SM70-NEXT: setp.ge.u32 %p2, %r7, %r1;
+; SM70-NEXT: sub.s32 %r3, %r8, %r2;
+; SM70-NEXT: sub.s32 %r4, %r7, %r1;
+; SM70-NEXT: selp.b32 %r5, %r4, %r7, %p2;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: selp.b32 %r6, %r3, %r8, %p1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p3 bra $L__BB162_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<7>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM70-NEXT: $L__BB163_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: setp.ge.u32 %p1, %r14, %r2;
+; SM70-NEXT: setp.ge.u32 %p2, %r13, %r1;
+; SM70-NEXT: sub.s32 %r5, %r14, %r2;
+; SM70-NEXT: sub.s32 %r6, %r13, %r1;
+; SM70-NEXT: selp.b32 %r7, %r6, %r13, %p2;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM70-NEXT: selp.b32 %r8, %r5, %r14, %p1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM70-NEXT: @%p3 bra $L__BB163_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM70-NEXT: $L__BB163_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r15;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r16;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: setp.ge.u32 %p4, %r16, %r4;
+; SM70-NEXT: setp.ge.u32 %p5, %r15, %r3;
+; SM70-NEXT: sub.s32 %r9, %r16, %r4;
+; SM70-NEXT: sub.s32 %r10, %r15, %r3;
+; SM70-NEXT: selp.b32 %r11, %r10, %r15, %p5;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: selp.b32 %r12, %r9, %r16, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p6, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd21;
+; SM70-NEXT: @%p6 bra $L__BB163_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<13>;
+; SM70-NEXT: .reg .b32 %r<33>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_cond_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM70-NEXT: $L__BB164_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r26;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: setp.ge.u32 %p1, %r26, %r6;
+; SM70-NEXT: setp.ge.u32 %p2, %r25, %r5;
+; SM70-NEXT: sub.s32 %r9, %r26, %r6;
+; SM70-NEXT: sub.s32 %r10, %r25, %r5;
+; SM70-NEXT: selp.b32 %r11, %r10, %r25, %p2;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r11;
+; SM70-NEXT: selp.b32 %r12, %r9, %r26, %p1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r12;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM70-NEXT: @%p3 bra $L__BB164_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM70-NEXT: $L__BB164_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r27;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: setp.ge.u32 %p4, %r28, %r8;
+; SM70-NEXT: setp.ge.u32 %p5, %r27, %r7;
+; SM70-NEXT: sub.s32 %r13, %r28, %r8;
+; SM70-NEXT: sub.s32 %r14, %r27, %r7;
+; SM70-NEXT: selp.b32 %r15, %r14, %r27, %p5;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM70-NEXT: selp.b32 %r16, %r13, %r28, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p6, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM70-NEXT: @%p6 bra $L__BB164_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd22;
+; SM70-NEXT: $L__BB164_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd23, %r29;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r30;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: setp.ge.u32 %p7, %r30, %r2;
+; SM70-NEXT: setp.ge.u32 %p8, %r29, %r1;
+; SM70-NEXT: sub.s32 %r17, %r30, %r2;
+; SM70-NEXT: sub.s32 %r18, %r29, %r1;
+; SM70-NEXT: selp.b32 %r19, %r18, %r29, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r19;
+; SM70-NEXT: selp.b32 %r20, %r17, %r30, %p7;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r20;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd26, %rd30;
+; SM70-NEXT: setp.ne.b64 %p9, %rd31, %rd26;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd31;
+; SM70-NEXT: @%p9 bra $L__BB164_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r31, %rd32;
+; SM70-NEXT: $L__BB164_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd33, %r31;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r32;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: setp.ge.u32 %p10, %r32, %r4;
+; SM70-NEXT: setp.ge.u32 %p11, %r31, %r3;
+; SM70-NEXT: sub.s32 %r21, %r32, %r4;
+; SM70-NEXT: sub.s32 %r22, %r31, %r3;
+; SM70-NEXT: selp.b32 %r23, %r22, %r31, %p11;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM70-NEXT: selp.b32 %r24, %r21, %r32, %p10;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd36, %rd40;
+; SM70-NEXT: setp.ne.b64 %p12, %rd41, %rd36;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r31, %rd41;
+; SM70-NEXT: @%p12 bra $L__BB164_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [usub_cond_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
+; SM70-NEXT: $L__BB165_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
+; SM70-NEXT: sub.s64 %rd6, %rd10, %rd1;
+; SM70-NEXT: selp.b64 %rd7, %rd6, %rd10, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM70-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
+; SM70-NEXT: mov.b64 %rd10, %rd3;
+; SM70-NEXT: @%p2 bra $L__BB165_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
+; SM70-NEXT: $L__BB165_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
+; SM70-NEXT: sub.s64 %rd8, %rd11, %rd2;
+; SM70-NEXT: selp.b64 %rd9, %rd8, %rd11, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM70-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
+; SM70-NEXT: mov.b64 %rd11, %rd4;
+; SM70-NEXT: @%p4 bra $L__BB165_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [usub_cond_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
+; SM70-NEXT: $L__BB166_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd18;
+; SM70-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
+; SM70-NEXT: sub.s64 %rd10, %rd5, %rd3;
+; SM70-NEXT: selp.b64 %rd11, %rd10, %rd5, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
+; SM70-NEXT: @%p2 bra $L__BB166_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
+; SM70-NEXT: $L__BB166_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd19;
+; SM70-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
+; SM70-NEXT: sub.s64 %rd12, %rd6, %rd4;
+; SM70-NEXT: selp.b64 %rd13, %rd12, %rd6, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
+; SM70-NEXT: @%p4 bra $L__BB166_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
+; SM70-NEXT: $L__BB166_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
+; SM70-NEXT: sub.s64 %rd14, %rd20, %rd1;
+; SM70-NEXT: selp.b64 %rd15, %rd14, %rd20, %p5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM70-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
+; SM70-NEXT: mov.b64 %rd20, %rd7;
+; SM70-NEXT: @%p6 bra $L__BB166_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
+; SM70-NEXT: $L__BB166_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
+; SM70-NEXT: sub.s64 %rd16, %rd21, %rd2;
+; SM70-NEXT: selp.b64 %rd17, %rd16, %rd21, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM70-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
+; SM70-NEXT: mov.b64 %rd21, %rd8;
+; SM70-NEXT: @%p8 bra $L__BB166_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<17>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [usub_cond_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [usub_cond_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
+; SM70-NEXT: $L__BB167_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd34;
+; SM70-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
+; SM70-NEXT: sub.s64 %rd18, %rd7, %rd5;
+; SM70-NEXT: selp.b64 %rd19, %rd18, %rd7, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
+; SM70-NEXT: @%p2 bra $L__BB167_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
+; SM70-NEXT: $L__BB167_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd35;
+; SM70-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
+; SM70-NEXT: sub.s64 %rd20, %rd8, %rd6;
+; SM70-NEXT: selp.b64 %rd21, %rd20, %rd8, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB167_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
+; SM70-NEXT: $L__BB167_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd36;
+; SM70-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
+; SM70-NEXT: sub.s64 %rd22, %rd9, %rd3;
+; SM70-NEXT: selp.b64 %rd23, %rd22, %rd9, %p5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
+; SM70-NEXT: @%p6 bra $L__BB167_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
+; SM70-NEXT: $L__BB167_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd37;
+; SM70-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
+; SM70-NEXT: sub.s64 %rd24, %rd10, %rd4;
+; SM70-NEXT: selp.b64 %rd25, %rd24, %rd10, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
+; SM70-NEXT: @%p8 bra $L__BB167_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
+; SM70-NEXT: $L__BB167_9: // %atomicrmw.start41
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd38;
+; SM70-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
+; SM70-NEXT: sub.s64 %rd26, %rd13, %rd1;
+; SM70-NEXT: selp.b64 %rd27, %rd26, %rd13, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM70-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
+; SM70-NEXT: @%p10 bra $L__BB167_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end40
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
+; SM70-NEXT: $L__BB167_11: // %atomicrmw.start47
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd39;
+; SM70-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
+; SM70-NEXT: sub.s64 %rd28, %rd14, %rd2;
+; SM70-NEXT: selp.b64 %rd29, %rd28, %rd14, %p11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM70-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
+; SM70-NEXT: @%p12 bra $L__BB167_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end46
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
+; SM70-NEXT: $L__BB167_13: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
+; SM70-NEXT: sub.s64 %rd30, %rd40, %rd11;
+; SM70-NEXT: selp.b64 %rd31, %rd30, %rd40, %p13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM70-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
+; SM70-NEXT: mov.b64 %rd40, %rd15;
+; SM70-NEXT: @%p14 bra $L__BB167_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end57
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
+; SM70-NEXT: $L__BB167_15: // %atomicrmw.start64
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
+; SM70-NEXT: sub.s64 %rd32, %rd41, %rd12;
+; SM70-NEXT: selp.b64 %rd33, %rd32, %rd41, %p15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM70-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
+; SM70-NEXT: mov.b64 %rd41, %rd16;
+; SM70-NEXT: @%p16 bra $L__BB167_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end63
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<22>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_sat_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r21, [%rd1];
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM70-NEXT: $L__BB168_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r21, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM70-NEXT: sub.s16 %rs9, %rs6, %rs8;
+; SM70-NEXT: setp.gt.u16 %p1, %rs9, %rs6;
+; SM70-NEXT: selp.b16 %rs10, -1, 0, %p1;
+; SM70-NEXT: sub.s16 %rs11, %rs5, %rs7;
+; SM70-NEXT: setp.gt.u16 %p2, %rs11, %rs5;
+; SM70-NEXT: selp.b16 %rs12, -1, 0, %p2;
+; SM70-NEXT: mov.b32 %r12, {%rs12, %rs10};
+; SM70-NEXT: xor.b32 %r13, %r12, -1;
+; SM70-NEXT: mov.b32 %r14, {%rs11, %rs9};
+; SM70-NEXT: and.b32 %r15, %r14, %r13;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r15;
+; SM70-NEXT: shl.b16 %rs15, %rs14, 8;
+; SM70-NEXT: or.b16 %rs16, %rs13, %rs15;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM70-NEXT: shl.b32 %r17, %r16, %r1;
+; SM70-NEXT: and.b32 %r18, %r21, %r2;
+; SM70-NEXT: or.b32 %r19, %r18, %r17;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r21, %r19;
+; SM70-NEXT: setp.ne.b32 %p3, %r3, %r21;
+; SM70-NEXT: mov.b32 %r21, %r3;
+; SM70-NEXT: @%p3 bra $L__BB168_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r20, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r20;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r13;
+; SM70-NEXT: $L__BB169_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM70-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r7;
+; SM70-NEXT: max.u16 %rs7, %rs6, %rs5;
+; SM70-NEXT: sub.s16 %rs8, %rs7, %rs5;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs8;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r11;
+; SM70-NEXT: max.u16 %rs11, %rs10, %rs9;
+; SM70-NEXT: sub.s16 %rs12, %rs11, %rs9;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r14;
+; SM70-NEXT: max.u16 %rs15, %rs14, %rs13;
+; SM70-NEXT: sub.s16 %rs16, %rs15, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs16;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB169_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<33>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB170_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM70-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r7;
+; SM70-NEXT: max.u16 %rs7, %rs6, %rs5;
+; SM70-NEXT: sub.s16 %rs8, %rs7, %rs5;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs8;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r11;
+; SM70-NEXT: max.u16 %rs11, %rs10, %rs9;
+; SM70-NEXT: sub.s16 %rs12, %rs11, %rs9;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r14;
+; SM70-NEXT: max.u16 %rs15, %rs14, %rs13;
+; SM70-NEXT: sub.s16 %rs16, %rs15, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs16;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs18, %r19;
+; SM70-NEXT: max.u16 %rs19, %rs18, %rs17;
+; SM70-NEXT: sub.s16 %rs20, %rs19, %rs17;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM70-NEXT: cvt.u16.u32 %rs21, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r22;
+; SM70-NEXT: max.u16 %rs23, %rs22, %rs21;
+; SM70-NEXT: sub.s16 %rs24, %rs23, %rs21;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs24;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs25, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs26, %r26;
+; SM70-NEXT: max.u16 %rs27, %rs26, %rs25;
+; SM70-NEXT: sub.s16 %rs28, %rs27, %rs25;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs28;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs29, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs30, %r29;
+; SM70-NEXT: max.u16 %rs31, %rs30, %rs29;
+; SM70-NEXT: sub.s16 %rs32, %rs31, %rs29;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs32;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB170_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB171_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r7;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: setp.gt.u16 %p1, %rs5, %rs4;
+; SM70-NEXT: selp.b16 %rs7, -1, 0, %p1;
+; SM70-NEXT: setp.gt.u16 %p2, %rs6, %rs3;
+; SM70-NEXT: selp.b16 %rs8, -1, 0, %p2;
+; SM70-NEXT: mov.b32 %r4, {%rs8, %rs7};
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: and.b32 %r6, %r3, %r5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r7;
+; SM70-NEXT: mov.b32 %r7, %r1;
+; SM70-NEXT: @%p3 bra $L__BB171_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM70-NEXT: $L__BB172_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r11;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: setp.gt.u16 %p1, %rs5, %rs4;
+; SM70-NEXT: selp.b16 %rs7, -1, 0, %p1;
+; SM70-NEXT: setp.gt.u16 %p2, %rs6, %rs3;
+; SM70-NEXT: selp.b16 %rs8, -1, 0, %p2;
+; SM70-NEXT: mov.b32 %r4, {%rs8, %rs7};
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: and.b32 %r6, %r3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r6;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r12;
+; SM70-NEXT: sub.s16 %rs13, %rs12, %rs10;
+; SM70-NEXT: sub.s16 %rs14, %rs11, %rs9;
+; SM70-NEXT: mov.b32 %r7, {%rs14, %rs13};
+; SM70-NEXT: setp.gt.u16 %p3, %rs13, %rs12;
+; SM70-NEXT: selp.b16 %rs15, -1, 0, %p3;
+; SM70-NEXT: setp.gt.u16 %p4, %rs14, %rs11;
+; SM70-NEXT: selp.b16 %rs16, -1, 0, %p4;
+; SM70-NEXT: mov.b32 %r8, {%rs16, %rs15};
+; SM70-NEXT: xor.b32 %r9, %r8, -1;
+; SM70-NEXT: and.b32 %r10, %r7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r12;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB172_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<33>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM70-NEXT: $L__BB173_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r21;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: setp.gt.u16 %p1, %rs5, %rs4;
+; SM70-NEXT: selp.b16 %rs7, -1, 0, %p1;
+; SM70-NEXT: setp.gt.u16 %p2, %rs6, %rs3;
+; SM70-NEXT: selp.b16 %rs8, -1, 0, %p2;
+; SM70-NEXT: mov.b32 %r6, {%rs8, %rs7};
+; SM70-NEXT: xor.b32 %r7, %r6, -1;
+; SM70-NEXT: and.b32 %r8, %r5, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r8;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r22;
+; SM70-NEXT: sub.s16 %rs13, %rs12, %rs10;
+; SM70-NEXT: sub.s16 %rs14, %rs11, %rs9;
+; SM70-NEXT: mov.b32 %r9, {%rs14, %rs13};
+; SM70-NEXT: setp.gt.u16 %p3, %rs13, %rs12;
+; SM70-NEXT: selp.b16 %rs15, -1, 0, %p3;
+; SM70-NEXT: setp.gt.u16 %p4, %rs14, %rs11;
+; SM70-NEXT: selp.b16 %rs16, -1, 0, %p4;
+; SM70-NEXT: mov.b32 %r10, {%rs16, %rs15};
+; SM70-NEXT: xor.b32 %r11, %r10, -1;
+; SM70-NEXT: and.b32 %r12, %r9, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r22;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB173_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd12;
+; SM70-NEXT: $L__BB173_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r23;
+; SM70-NEXT: sub.s16 %rs21, %rs20, %rs18;
+; SM70-NEXT: sub.s16 %rs22, %rs19, %rs17;
+; SM70-NEXT: mov.b32 %r13, {%rs22, %rs21};
+; SM70-NEXT: setp.gt.u16 %p6, %rs21, %rs20;
+; SM70-NEXT: selp.b16 %rs23, -1, 0, %p6;
+; SM70-NEXT: setp.gt.u16 %p7, %rs22, %rs19;
+; SM70-NEXT: selp.b16 %rs24, -1, 0, %p7;
+; SM70-NEXT: mov.b32 %r14, {%rs24, %rs23};
+; SM70-NEXT: xor.b32 %r15, %r14, -1;
+; SM70-NEXT: and.b32 %r16, %r13, %r15;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r16;
+; SM70-NEXT: mov.b32 {%rs25, %rs26}, %r4;
+; SM70-NEXT: mov.b32 {%rs27, %rs28}, %r24;
+; SM70-NEXT: sub.s16 %rs29, %rs28, %rs26;
+; SM70-NEXT: sub.s16 %rs30, %rs27, %rs25;
+; SM70-NEXT: mov.b32 %r17, {%rs30, %rs29};
+; SM70-NEXT: setp.gt.u16 %p8, %rs29, %rs28;
+; SM70-NEXT: selp.b16 %rs31, -1, 0, %p8;
+; SM70-NEXT: setp.gt.u16 %p9, %rs30, %rs27;
+; SM70-NEXT: selp.b16 %rs32, -1, 0, %p9;
+; SM70-NEXT: mov.b32 %r18, {%rs32, %rs31};
+; SM70-NEXT: xor.b32 %r19, %r18, -1;
+; SM70-NEXT: and.b32 %r20, %r17, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r20;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r24;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB173_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB174_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r3, %r7, %r1;
+; SM70-NEXT: sub.s32 %r4, %r3, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r4;
+; SM70-NEXT: max.u32 %r5, %r8, %r2;
+; SM70-NEXT: sub.s32 %r6, %r5, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB174_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM70-NEXT: $L__BB175_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r5, %r13, %r1;
+; SM70-NEXT: sub.s32 %r6, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r6;
+; SM70-NEXT: max.u32 %r7, %r14, %r2;
+; SM70-NEXT: sub.s32 %r8, %r7, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB175_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM70-NEXT: $L__BB175_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r9, %r15, %r3;
+; SM70-NEXT: sub.s32 %r10, %r9, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r10;
+; SM70-NEXT: max.u32 %r11, %r16, %r4;
+; SM70-NEXT: sub.s32 %r12, %r11, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB175_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<33>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_sat_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM70-NEXT: $L__BB176_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r9, %r25, %r5;
+; SM70-NEXT: sub.s32 %r10, %r9, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r10;
+; SM70-NEXT: max.u32 %r11, %r26, %r6;
+; SM70-NEXT: sub.s32 %r12, %r11, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB176_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM70-NEXT: $L__BB176_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r13, %r27, %r7;
+; SM70-NEXT: sub.s32 %r14, %r13, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r14;
+; SM70-NEXT: max.u32 %r15, %r28, %r8;
+; SM70-NEXT: sub.s32 %r16, %r15, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r16;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB176_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd22;
+; SM70-NEXT: $L__BB176_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r17, %r29, %r1;
+; SM70-NEXT: sub.s32 %r18, %r17, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r18;
+; SM70-NEXT: max.u32 %r19, %r30, %r2;
+; SM70-NEXT: sub.s32 %r20, %r19, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r20;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r29;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r30;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM70-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd31;
+; SM70-NEXT: @%p3 bra $L__BB176_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r31, %rd32;
+; SM70-NEXT: $L__BB176_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r21, %r31, %r3;
+; SM70-NEXT: sub.s32 %r22, %r21, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r22;
+; SM70-NEXT: max.u32 %r23, %r32, %r4;
+; SM70-NEXT: sub.s32 %r24, %r23, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r24;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r31;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r32;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM70-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r31, %rd41;
+; SM70-NEXT: @%p4 bra $L__BB176_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [usub_sat_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
+; SM70-NEXT: $L__BB177_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd6, %rd10, %rd1;
+; SM70-NEXT: sub.s64 %rd7, %rd6, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
+; SM70-NEXT: mov.b64 %rd10, %rd3;
+; SM70-NEXT: @%p1 bra $L__BB177_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
+; SM70-NEXT: $L__BB177_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd8, %rd11, %rd2;
+; SM70-NEXT: sub.s64 %rd9, %rd8, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
+; SM70-NEXT: mov.b64 %rd11, %rd4;
+; SM70-NEXT: @%p2 bra $L__BB177_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [usub_sat_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
+; SM70-NEXT: $L__BB178_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd18;
+; SM70-NEXT: max.u64 %rd10, %rd5, %rd3;
+; SM70-NEXT: sub.s64 %rd11, %rd10, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
+; SM70-NEXT: @%p1 bra $L__BB178_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
+; SM70-NEXT: $L__BB178_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd19;
+; SM70-NEXT: max.u64 %rd12, %rd6, %rd4;
+; SM70-NEXT: sub.s64 %rd13, %rd12, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM70-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
+; SM70-NEXT: @%p2 bra $L__BB178_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
+; SM70-NEXT: $L__BB178_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd14, %rd20, %rd1;
+; SM70-NEXT: sub.s64 %rd15, %rd14, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
+; SM70-NEXT: mov.b64 %rd20, %rd7;
+; SM70-NEXT: @%p3 bra $L__BB178_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
+; SM70-NEXT: $L__BB178_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd16, %rd21, %rd2;
+; SM70-NEXT: sub.s64 %rd17, %rd16, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
+; SM70-NEXT: mov.b64 %rd21, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB178_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [usub_sat_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [usub_sat_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
+; SM70-NEXT: $L__BB179_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd34;
+; SM70-NEXT: max.u64 %rd18, %rd7, %rd5;
+; SM70-NEXT: sub.s64 %rd19, %rd18, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM70-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
+; SM70-NEXT: @%p1 bra $L__BB179_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
+; SM70-NEXT: $L__BB179_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd35;
+; SM70-NEXT: max.u64 %rd20, %rd8, %rd6;
+; SM70-NEXT: sub.s64 %rd21, %rd20, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM70-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
+; SM70-NEXT: @%p2 bra $L__BB179_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
+; SM70-NEXT: $L__BB179_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd36;
+; SM70-NEXT: max.u64 %rd22, %rd9, %rd3;
+; SM70-NEXT: sub.s64 %rd23, %rd22, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM70-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
+; SM70-NEXT: @%p3 bra $L__BB179_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
+; SM70-NEXT: $L__BB179_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd37;
+; SM70-NEXT: max.u64 %rd24, %rd10, %rd4;
+; SM70-NEXT: sub.s64 %rd25, %rd24, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM70-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
+; SM70-NEXT: @%p4 bra $L__BB179_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
+; SM70-NEXT: $L__BB179_9: // %atomicrmw.start41
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd38;
+; SM70-NEXT: max.u64 %rd26, %rd13, %rd1;
+; SM70-NEXT: sub.s64 %rd27, %rd26, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM70-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
+; SM70-NEXT: @%p5 bra $L__BB179_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end40
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
+; SM70-NEXT: $L__BB179_11: // %atomicrmw.start47
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd39;
+; SM70-NEXT: max.u64 %rd28, %rd14, %rd2;
+; SM70-NEXT: sub.s64 %rd29, %rd28, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM70-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
+; SM70-NEXT: @%p6 bra $L__BB179_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end46
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
+; SM70-NEXT: $L__BB179_13: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd30, %rd40, %rd11;
+; SM70-NEXT: sub.s64 %rd31, %rd30, %rd11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
+; SM70-NEXT: mov.b64 %rd40, %rd15;
+; SM70-NEXT: @%p7 bra $L__BB179_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end57
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
+; SM70-NEXT: $L__BB179_15: // %atomicrmw.start64
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd32, %rd41, %rd12;
+; SM70-NEXT: sub.s64 %rd33, %rd32, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
+; SM70-NEXT: mov.b64 %rd41, %rd16;
+; SM70-NEXT: @%p8 bra $L__BB179_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end63
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM70-LABEL: fadd_acq_rel_v2f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM70-LABEL: fadd_acq_rel_v4f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM70-LABEL: fadd_acq_rel_v8f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM70-LABEL: fsub_acq_rel_v2f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB183_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: sub.rn.f32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB183_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM70-LABEL: fsub_acq_rel_v4f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB184_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: sub.rn.f32 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB184_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB184_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: sub.rn.f32 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB184_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM70-LABEL: fsub_acq_rel_v8f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB185_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r9, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: sub.rn.f32 %r10, %r18, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB185_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM70-NEXT: $L__BB185_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r11, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: sub.rn.f32 %r12, %r20, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB185_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM70-NEXT: $L__BB185_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r13, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r13;
+; SM70-NEXT: sub.rn.f32 %r14, %r22, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM70-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM70-NEXT: @%p3 bra $L__BB185_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM70-NEXT: $L__BB185_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r15, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM70-NEXT: sub.rn.f32 %r16, %r24, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM70-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM70-NEXT: @%p4 bra $L__BB185_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM70-LABEL: fmin_acq_rel_v2f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v2f32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB186_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: min.f32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB186_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM70-LABEL: fmin_acq_rel_v4f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v4f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB187_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: min.f32 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB187_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB187_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: min.f32 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB187_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM70-LABEL: fmin_acq_rel_v8f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmin_acq_rel_v8f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB188_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r9, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: min.f32 %r10, %r18, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB188_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM70-NEXT: $L__BB188_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r11, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: min.f32 %r12, %r20, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB188_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM70-NEXT: $L__BB188_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r13, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r13;
+; SM70-NEXT: min.f32 %r14, %r22, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM70-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM70-NEXT: @%p3 bra $L__BB188_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM70-NEXT: $L__BB188_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r15, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM70-NEXT: min.f32 %r16, %r24, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM70-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM70-NEXT: @%p4 bra $L__BB188_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM70-LABEL: fmax_acq_rel_v2f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v2f32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB189_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: max.f32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB189_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM70-LABEL: fmax_acq_rel_v4f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v4f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB190_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: max.f32 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB190_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB190_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: max.f32 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB190_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM70-LABEL: fmax_acq_rel_v8f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmax_acq_rel_v8f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB191_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r9, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: max.f32 %r10, %r18, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB191_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM70-NEXT: $L__BB191_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r11, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: max.f32 %r12, %r20, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB191_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM70-NEXT: $L__BB191_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r13, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r13;
+; SM70-NEXT: max.f32 %r14, %r22, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM70-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM70-NEXT: @%p3 bra $L__BB191_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM70-NEXT: $L__BB191_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r15, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM70-NEXT: max.f32 %r16, %r24, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM70-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM70-NEXT: @%p4 bra $L__BB191_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM70-LABEL: fminimum_acq_rel_v2f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v2f32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM70-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
+; SM70-NEXT: $L__BB192_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r13, %r1;
+; SM70-NEXT: min.f32 %r3, %r13, %r1;
+; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r13, -2147483648;
+; SM70-NEXT: selp.f32 %r5, %r13, %r4, %p2;
+; SM70-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: setp.nan.f32 %p5, %r14, %r2;
+; SM70-NEXT: min.f32 %r8, %r14, %r2;
+; SM70-NEXT: selp.f32 %r9, 0f7FC00000, %r8, %p5;
+; SM70-NEXT: setp.eq.b32 %p6, %r14, -2147483648;
+; SM70-NEXT: selp.f32 %r10, %r14, %r9, %p6;
+; SM70-NEXT: selp.f32 %r11, %r2, %r10, %p7;
+; SM70-NEXT: setp.eq.f32 %p8, %r9, 0f00000000;
+; SM70-NEXT: selp.f32 %r12, %r11, %r9, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB192_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM70-LABEL: fminimum_acq_rel_v4f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<19>;
+; SM70-NEXT: .reg .b32 %r<29>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM70-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
+; SM70-NEXT: $L__BB193_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r25, %r1;
+; SM70-NEXT: min.f32 %r5, %r25, %r1;
+; SM70-NEXT: selp.f32 %r6, 0f7FC00000, %r5, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r25, -2147483648;
+; SM70-NEXT: selp.f32 %r7, %r25, %r6, %p2;
+; SM70-NEXT: selp.f32 %r8, %r1, %r7, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r6, 0f00000000;
+; SM70-NEXT: selp.f32 %r9, %r8, %r6, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: setp.nan.f32 %p5, %r26, %r2;
+; SM70-NEXT: min.f32 %r10, %r26, %r2;
+; SM70-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p5;
+; SM70-NEXT: setp.eq.b32 %p6, %r26, -2147483648;
+; SM70-NEXT: selp.f32 %r12, %r26, %r11, %p6;
+; SM70-NEXT: selp.f32 %r13, %r2, %r12, %p7;
+; SM70-NEXT: setp.eq.f32 %p8, %r11, 0f00000000;
+; SM70-NEXT: selp.f32 %r14, %r13, %r11, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB193_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM70-NEXT: setp.eq.b32 %p12, %r3, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p16, %r4, -2147483648;
+; SM70-NEXT: $L__BB193_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p10, %r27, %r3;
+; SM70-NEXT: min.f32 %r15, %r27, %r3;
+; SM70-NEXT: selp.f32 %r16, 0f7FC00000, %r15, %p10;
+; SM70-NEXT: setp.eq.b32 %p11, %r27, -2147483648;
+; SM70-NEXT: selp.f32 %r17, %r27, %r16, %p11;
+; SM70-NEXT: selp.f32 %r18, %r3, %r17, %p12;
+; SM70-NEXT: setp.eq.f32 %p13, %r16, 0f00000000;
+; SM70-NEXT: selp.f32 %r19, %r18, %r16, %p13;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r19;
+; SM70-NEXT: setp.nan.f32 %p14, %r28, %r4;
+; SM70-NEXT: min.f32 %r20, %r28, %r4;
+; SM70-NEXT: selp.f32 %r21, 0f7FC00000, %r20, %p14;
+; SM70-NEXT: setp.eq.b32 %p15, %r28, -2147483648;
+; SM70-NEXT: selp.f32 %r22, %r28, %r21, %p15;
+; SM70-NEXT: selp.f32 %r23, %r4, %r22, %p16;
+; SM70-NEXT: setp.eq.f32 %p17, %r21, 0f00000000;
+; SM70-NEXT: selp.f32 %r24, %r23, %r21, %p17;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r24;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM70-NEXT: @%p18 bra $L__BB193_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM70-LABEL: fminimum_acq_rel_v8f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<37>;
+; SM70-NEXT: .reg .b32 %r<57>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM70-NEXT: setp.eq.b32 %p3, %r5, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p7, %r6, -2147483648;
+; SM70-NEXT: $L__BB194_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r49, %r5;
+; SM70-NEXT: min.f32 %r9, %r49, %r5;
+; SM70-NEXT: selp.f32 %r10, 0f7FC00000, %r9, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r49, -2147483648;
+; SM70-NEXT: selp.f32 %r11, %r49, %r10, %p2;
+; SM70-NEXT: selp.f32 %r12, %r5, %r11, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r10, 0f00000000;
+; SM70-NEXT: selp.f32 %r13, %r12, %r10, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM70-NEXT: setp.nan.f32 %p5, %r50, %r6;
+; SM70-NEXT: min.f32 %r14, %r50, %r6;
+; SM70-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p5;
+; SM70-NEXT: setp.eq.b32 %p6, %r50, -2147483648;
+; SM70-NEXT: selp.f32 %r16, %r50, %r15, %p6;
+; SM70-NEXT: selp.f32 %r17, %r6, %r16, %p7;
+; SM70-NEXT: setp.eq.f32 %p8, %r15, 0f00000000;
+; SM70-NEXT: selp.f32 %r18, %r17, %r15, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r49;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r50;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB194_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r51, %rd12;
+; SM70-NEXT: setp.eq.b32 %p12, %r7, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p16, %r8, -2147483648;
+; SM70-NEXT: $L__BB194_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p10, %r51, %r7;
+; SM70-NEXT: min.f32 %r19, %r51, %r7;
+; SM70-NEXT: selp.f32 %r20, 0f7FC00000, %r19, %p10;
+; SM70-NEXT: setp.eq.b32 %p11, %r51, -2147483648;
+; SM70-NEXT: selp.f32 %r21, %r51, %r20, %p11;
+; SM70-NEXT: selp.f32 %r22, %r7, %r21, %p12;
+; SM70-NEXT: setp.eq.f32 %p13, %r20, 0f00000000;
+; SM70-NEXT: selp.f32 %r23, %r22, %r20, %p13;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r23;
+; SM70-NEXT: setp.nan.f32 %p14, %r52, %r8;
+; SM70-NEXT: min.f32 %r24, %r52, %r8;
+; SM70-NEXT: selp.f32 %r25, 0f7FC00000, %r24, %p14;
+; SM70-NEXT: setp.eq.b32 %p15, %r52, -2147483648;
+; SM70-NEXT: selp.f32 %r26, %r52, %r25, %p15;
+; SM70-NEXT: selp.f32 %r27, %r8, %r26, %p16;
+; SM70-NEXT: setp.eq.f32 %p17, %r25, 0f00000000;
+; SM70-NEXT: selp.f32 %r28, %r27, %r25, %p17;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r51;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r52;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r51, %rd21;
+; SM70-NEXT: @%p18 bra $L__BB194_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r53, %rd22;
+; SM70-NEXT: setp.eq.b32 %p21, %r1, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p25, %r2, -2147483648;
+; SM70-NEXT: $L__BB194_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p19, %r53, %r1;
+; SM70-NEXT: min.f32 %r29, %r53, %r1;
+; SM70-NEXT: selp.f32 %r30, 0f7FC00000, %r29, %p19;
+; SM70-NEXT: setp.eq.b32 %p20, %r53, -2147483648;
+; SM70-NEXT: selp.f32 %r31, %r53, %r30, %p20;
+; SM70-NEXT: selp.f32 %r32, %r1, %r31, %p21;
+; SM70-NEXT: setp.eq.f32 %p22, %r30, 0f00000000;
+; SM70-NEXT: selp.f32 %r33, %r32, %r30, %p22;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r33;
+; SM70-NEXT: setp.nan.f32 %p23, %r54, %r2;
+; SM70-NEXT: min.f32 %r34, %r54, %r2;
+; SM70-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
+; SM70-NEXT: setp.eq.b32 %p24, %r54, -2147483648;
+; SM70-NEXT: selp.f32 %r36, %r54, %r35, %p24;
+; SM70-NEXT: selp.f32 %r37, %r2, %r36, %p25;
+; SM70-NEXT: setp.eq.f32 %p26, %r35, 0f00000000;
+; SM70-NEXT: selp.f32 %r38, %r37, %r35, %p26;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r38;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r53;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r54;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM70-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r53, %rd31;
+; SM70-NEXT: @%p27 bra $L__BB194_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r55, %rd32;
+; SM70-NEXT: setp.eq.b32 %p30, %r3, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p34, %r4, -2147483648;
+; SM70-NEXT: $L__BB194_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p28, %r55, %r3;
+; SM70-NEXT: min.f32 %r39, %r55, %r3;
+; SM70-NEXT: selp.f32 %r40, 0f7FC00000, %r39, %p28;
+; SM70-NEXT: setp.eq.b32 %p29, %r55, -2147483648;
+; SM70-NEXT: selp.f32 %r41, %r55, %r40, %p29;
+; SM70-NEXT: selp.f32 %r42, %r3, %r41, %p30;
+; SM70-NEXT: setp.eq.f32 %p31, %r40, 0f00000000;
+; SM70-NEXT: selp.f32 %r43, %r42, %r40, %p31;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r43;
+; SM70-NEXT: setp.nan.f32 %p32, %r56, %r4;
+; SM70-NEXT: min.f32 %r44, %r56, %r4;
+; SM70-NEXT: selp.f32 %r45, 0f7FC00000, %r44, %p32;
+; SM70-NEXT: setp.eq.b32 %p33, %r56, -2147483648;
+; SM70-NEXT: selp.f32 %r46, %r56, %r45, %p33;
+; SM70-NEXT: selp.f32 %r47, %r4, %r46, %p34;
+; SM70-NEXT: setp.eq.f32 %p35, %r45, 0f00000000;
+; SM70-NEXT: selp.f32 %r48, %r47, %r45, %p35;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r48;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r55;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r56;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM70-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r55, %rd41;
+; SM70-NEXT: @%p36 bra $L__BB194_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r49, %r50, %r51, %r52};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v2f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v2f32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM70-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM70-NEXT: setp.eq.b32 %p7, %r2, 0;
+; SM70-NEXT: $L__BB195_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r13, %r1;
+; SM70-NEXT: max.f32 %r3, %r13, %r1;
+; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r13, 0;
+; SM70-NEXT: selp.f32 %r5, %r13, %r4, %p2;
+; SM70-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: setp.nan.f32 %p5, %r14, %r2;
+; SM70-NEXT: max.f32 %r8, %r14, %r2;
+; SM70-NEXT: selp.f32 %r9, 0f7FC00000, %r8, %p5;
+; SM70-NEXT: setp.eq.b32 %p6, %r14, 0;
+; SM70-NEXT: selp.f32 %r10, %r14, %r9, %p6;
+; SM70-NEXT: selp.f32 %r11, %r2, %r10, %p7;
+; SM70-NEXT: setp.eq.f32 %p8, %r9, 0f00000000;
+; SM70-NEXT: selp.f32 %r12, %r11, %r9, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB195_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v4f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<19>;
+; SM70-NEXT: .reg .b32 %r<29>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM70-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM70-NEXT: setp.eq.b32 %p7, %r2, 0;
+; SM70-NEXT: $L__BB196_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r25, %r1;
+; SM70-NEXT: max.f32 %r5, %r25, %r1;
+; SM70-NEXT: selp.f32 %r6, 0f7FC00000, %r5, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r25, 0;
+; SM70-NEXT: selp.f32 %r7, %r25, %r6, %p2;
+; SM70-NEXT: selp.f32 %r8, %r1, %r7, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r6, 0f00000000;
+; SM70-NEXT: selp.f32 %r9, %r8, %r6, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: setp.nan.f32 %p5, %r26, %r2;
+; SM70-NEXT: max.f32 %r10, %r26, %r2;
+; SM70-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p5;
+; SM70-NEXT: setp.eq.b32 %p6, %r26, 0;
+; SM70-NEXT: selp.f32 %r12, %r26, %r11, %p6;
+; SM70-NEXT: selp.f32 %r13, %r2, %r12, %p7;
+; SM70-NEXT: setp.eq.f32 %p8, %r11, 0f00000000;
+; SM70-NEXT: selp.f32 %r14, %r13, %r11, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB196_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM70-NEXT: setp.eq.b32 %p12, %r3, 0;
+; SM70-NEXT: setp.eq.b32 %p16, %r4, 0;
+; SM70-NEXT: $L__BB196_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p10, %r27, %r3;
+; SM70-NEXT: max.f32 %r15, %r27, %r3;
+; SM70-NEXT: selp.f32 %r16, 0f7FC00000, %r15, %p10;
+; SM70-NEXT: setp.eq.b32 %p11, %r27, 0;
+; SM70-NEXT: selp.f32 %r17, %r27, %r16, %p11;
+; SM70-NEXT: selp.f32 %r18, %r3, %r17, %p12;
+; SM70-NEXT: setp.eq.f32 %p13, %r16, 0f00000000;
+; SM70-NEXT: selp.f32 %r19, %r18, %r16, %p13;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r19;
+; SM70-NEXT: setp.nan.f32 %p14, %r28, %r4;
+; SM70-NEXT: max.f32 %r20, %r28, %r4;
+; SM70-NEXT: selp.f32 %r21, 0f7FC00000, %r20, %p14;
+; SM70-NEXT: setp.eq.b32 %p15, %r28, 0;
+; SM70-NEXT: selp.f32 %r22, %r28, %r21, %p15;
+; SM70-NEXT: selp.f32 %r23, %r4, %r22, %p16;
+; SM70-NEXT: setp.eq.f32 %p17, %r21, 0f00000000;
+; SM70-NEXT: selp.f32 %r24, %r23, %r21, %p17;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r24;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM70-NEXT: @%p18 bra $L__BB196_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v8f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<37>;
+; SM70-NEXT: .reg .b32 %r<57>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM70-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM70-NEXT: setp.eq.b32 %p7, %r6, 0;
+; SM70-NEXT: $L__BB197_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r49, %r5;
+; SM70-NEXT: max.f32 %r9, %r49, %r5;
+; SM70-NEXT: selp.f32 %r10, 0f7FC00000, %r9, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r49, 0;
+; SM70-NEXT: selp.f32 %r11, %r49, %r10, %p2;
+; SM70-NEXT: selp.f32 %r12, %r5, %r11, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r10, 0f00000000;
+; SM70-NEXT: selp.f32 %r13, %r12, %r10, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM70-NEXT: setp.nan.f32 %p5, %r50, %r6;
+; SM70-NEXT: max.f32 %r14, %r50, %r6;
+; SM70-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p5;
+; SM70-NEXT: setp.eq.b32 %p6, %r50, 0;
+; SM70-NEXT: selp.f32 %r16, %r50, %r15, %p6;
+; SM70-NEXT: selp.f32 %r17, %r6, %r16, %p7;
+; SM70-NEXT: setp.eq.f32 %p8, %r15, 0f00000000;
+; SM70-NEXT: selp.f32 %r18, %r17, %r15, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r49;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r50;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB197_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r51, %rd12;
+; SM70-NEXT: setp.eq.b32 %p12, %r7, 0;
+; SM70-NEXT: setp.eq.b32 %p16, %r8, 0;
+; SM70-NEXT: $L__BB197_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p10, %r51, %r7;
+; SM70-NEXT: max.f32 %r19, %r51, %r7;
+; SM70-NEXT: selp.f32 %r20, 0f7FC00000, %r19, %p10;
+; SM70-NEXT: setp.eq.b32 %p11, %r51, 0;
+; SM70-NEXT: selp.f32 %r21, %r51, %r20, %p11;
+; SM70-NEXT: selp.f32 %r22, %r7, %r21, %p12;
+; SM70-NEXT: setp.eq.f32 %p13, %r20, 0f00000000;
+; SM70-NEXT: selp.f32 %r23, %r22, %r20, %p13;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r23;
+; SM70-NEXT: setp.nan.f32 %p14, %r52, %r8;
+; SM70-NEXT: max.f32 %r24, %r52, %r8;
+; SM70-NEXT: selp.f32 %r25, 0f7FC00000, %r24, %p14;
+; SM70-NEXT: setp.eq.b32 %p15, %r52, 0;
+; SM70-NEXT: selp.f32 %r26, %r52, %r25, %p15;
+; SM70-NEXT: selp.f32 %r27, %r8, %r26, %p16;
+; SM70-NEXT: setp.eq.f32 %p17, %r25, 0f00000000;
+; SM70-NEXT: selp.f32 %r28, %r27, %r25, %p17;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r51;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r52;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r51, %rd21;
+; SM70-NEXT: @%p18 bra $L__BB197_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r53, %rd22;
+; SM70-NEXT: setp.eq.b32 %p21, %r1, 0;
+; SM70-NEXT: setp.eq.b32 %p25, %r2, 0;
+; SM70-NEXT: $L__BB197_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p19, %r53, %r1;
+; SM70-NEXT: max.f32 %r29, %r53, %r1;
+; SM70-NEXT: selp.f32 %r30, 0f7FC00000, %r29, %p19;
+; SM70-NEXT: setp.eq.b32 %p20, %r53, 0;
+; SM70-NEXT: selp.f32 %r31, %r53, %r30, %p20;
+; SM70-NEXT: selp.f32 %r32, %r1, %r31, %p21;
+; SM70-NEXT: setp.eq.f32 %p22, %r30, 0f00000000;
+; SM70-NEXT: selp.f32 %r33, %r32, %r30, %p22;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r33;
+; SM70-NEXT: setp.nan.f32 %p23, %r54, %r2;
+; SM70-NEXT: max.f32 %r34, %r54, %r2;
+; SM70-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
+; SM70-NEXT: setp.eq.b32 %p24, %r54, 0;
+; SM70-NEXT: selp.f32 %r36, %r54, %r35, %p24;
+; SM70-NEXT: selp.f32 %r37, %r2, %r36, %p25;
+; SM70-NEXT: setp.eq.f32 %p26, %r35, 0f00000000;
+; SM70-NEXT: selp.f32 %r38, %r37, %r35, %p26;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r38;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r53;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r54;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM70-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r53, %rd31;
+; SM70-NEXT: @%p27 bra $L__BB197_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r55, %rd32;
+; SM70-NEXT: setp.eq.b32 %p30, %r3, 0;
+; SM70-NEXT: setp.eq.b32 %p34, %r4, 0;
+; SM70-NEXT: $L__BB197_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p28, %r55, %r3;
+; SM70-NEXT: max.f32 %r39, %r55, %r3;
+; SM70-NEXT: selp.f32 %r40, 0f7FC00000, %r39, %p28;
+; SM70-NEXT: setp.eq.b32 %p29, %r55, 0;
+; SM70-NEXT: selp.f32 %r41, %r55, %r40, %p29;
+; SM70-NEXT: selp.f32 %r42, %r3, %r41, %p30;
+; SM70-NEXT: setp.eq.f32 %p31, %r40, 0f00000000;
+; SM70-NEXT: selp.f32 %r43, %r42, %r40, %p31;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r43;
+; SM70-NEXT: setp.nan.f32 %p32, %r56, %r4;
+; SM70-NEXT: max.f32 %r44, %r56, %r4;
+; SM70-NEXT: selp.f32 %r45, 0f7FC00000, %r44, %p32;
+; SM70-NEXT: setp.eq.b32 %p33, %r56, 0;
+; SM70-NEXT: selp.f32 %r46, %r56, %r45, %p33;
+; SM70-NEXT: selp.f32 %r47, %r4, %r46, %p34;
+; SM70-NEXT: setp.eq.f32 %p35, %r45, 0f00000000;
+; SM70-NEXT: selp.f32 %r48, %r47, %r45, %p35;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r48;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r55;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r56;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM70-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r55, %rd41;
+; SM70-NEXT: @%p36 bra $L__BB197_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r49, %r50, %r51, %r52};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x double> @fadd_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM70-LABEL: fadd_acq_rel_v2f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fadd_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM70-LABEL: fadd_acq_rel_v4f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fadd_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM70-LABEL: fadd_acq_rel_v8f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [fadd_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [fadd_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM70-LABEL: fsub_acq_rel_v2f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [fsub_acq_rel_v2f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd5];
+; SM70-NEXT: $L__BB201_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd6, %rd8, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM70-NEXT: mov.b64 %rd8, %rd3;
+; SM70-NEXT: @%p1 bra $L__BB201_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd5+8];
+; SM70-NEXT: $L__BB201_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd7, %rd9, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
+; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
+; SM70-NEXT: mov.b64 %rd9, %rd4;
+; SM70-NEXT: @%p2 bra $L__BB201_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM70-LABEL: fsub_acq_rel_v4f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [fsub_acq_rel_v4f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd9];
+; SM70-NEXT: $L__BB202_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd14;
+; SM70-NEXT: sub.rn.f64 %rd10, %rd5, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
+; SM70-NEXT: @%p1 bra $L__BB202_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd9+8];
+; SM70-NEXT: $L__BB202_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd15;
+; SM70-NEXT: sub.rn.f64 %rd11, %rd6, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
+; SM70-NEXT: @%p2 bra $L__BB202_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd9+16];
+; SM70-NEXT: $L__BB202_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd12, %rd16, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM70-NEXT: mov.b64 %rd16, %rd7;
+; SM70-NEXT: @%p3 bra $L__BB202_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd9+24];
+; SM70-NEXT: $L__BB202_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd13, %rd17, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
+; SM70-NEXT: mov.b64 %rd17, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB202_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM70-LABEL: fsub_acq_rel_v8f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<34>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [fsub_acq_rel_v8f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fsub_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fsub_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd17];
+; SM70-NEXT: $L__BB203_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd26;
+; SM70-NEXT: sub.rn.f64 %rd18, %rd7, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM70-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
+; SM70-NEXT: @%p1 bra $L__BB203_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd17+8];
+; SM70-NEXT: $L__BB203_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd27;
+; SM70-NEXT: sub.rn.f64 %rd19, %rd8, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
+; SM70-NEXT: @%p2 bra $L__BB203_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd17+16];
+; SM70-NEXT: $L__BB203_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd28;
+; SM70-NEXT: sub.rn.f64 %rd20, %rd9, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM70-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
+; SM70-NEXT: @%p3 bra $L__BB203_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd17+24];
+; SM70-NEXT: $L__BB203_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd29;
+; SM70-NEXT: sub.rn.f64 %rd21, %rd10, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
+; SM70-NEXT: @%p4 bra $L__BB203_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd17+32];
+; SM70-NEXT: $L__BB203_9: // %atomicrmw.start41
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd30;
+; SM70-NEXT: sub.rn.f64 %rd22, %rd13, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
+; SM70-NEXT: @%p5 bra $L__BB203_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end40
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd17+40];
+; SM70-NEXT: $L__BB203_11: // %atomicrmw.start47
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd31;
+; SM70-NEXT: sub.rn.f64 %rd23, %rd14, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
+; SM70-NEXT: @%p6 bra $L__BB203_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end46
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd17+48];
+; SM70-NEXT: $L__BB203_13: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd24, %rd32, %rd11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM70-NEXT: mov.b64 %rd32, %rd15;
+; SM70-NEXT: @%p7 bra $L__BB203_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end57
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd17+56];
+; SM70-NEXT: $L__BB203_15: // %atomicrmw.start64
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd25, %rd33, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
+; SM70-NEXT: mov.b64 %rd33, %rd16;
+; SM70-NEXT: @%p8 bra $L__BB203_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end63
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM70-LABEL: fmin_acq_rel_v2f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [fmin_acq_rel_v2f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd5];
+; SM70-NEXT: $L__BB204_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd6, %rd8, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM70-NEXT: mov.b64 %rd8, %rd3;
+; SM70-NEXT: @%p1 bra $L__BB204_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd5+8];
+; SM70-NEXT: $L__BB204_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd7, %rd9, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
+; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
+; SM70-NEXT: mov.b64 %rd9, %rd4;
+; SM70-NEXT: @%p2 bra $L__BB204_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM70-LABEL: fmin_acq_rel_v4f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [fmin_acq_rel_v4f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd9];
+; SM70-NEXT: $L__BB205_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd14;
+; SM70-NEXT: min.f64 %rd10, %rd5, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
+; SM70-NEXT: @%p1 bra $L__BB205_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd9+8];
+; SM70-NEXT: $L__BB205_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd15;
+; SM70-NEXT: min.f64 %rd11, %rd6, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
+; SM70-NEXT: @%p2 bra $L__BB205_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd9+16];
+; SM70-NEXT: $L__BB205_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd12, %rd16, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM70-NEXT: mov.b64 %rd16, %rd7;
+; SM70-NEXT: @%p3 bra $L__BB205_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd9+24];
+; SM70-NEXT: $L__BB205_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd13, %rd17, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
+; SM70-NEXT: mov.b64 %rd17, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB205_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM70-LABEL: fmin_acq_rel_v8f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<34>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [fmin_acq_rel_v8f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmin_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmin_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd17];
+; SM70-NEXT: $L__BB206_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd26;
+; SM70-NEXT: min.f64 %rd18, %rd7, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM70-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
+; SM70-NEXT: @%p1 bra $L__BB206_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd17+8];
+; SM70-NEXT: $L__BB206_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd27;
+; SM70-NEXT: min.f64 %rd19, %rd8, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
+; SM70-NEXT: @%p2 bra $L__BB206_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd17+16];
+; SM70-NEXT: $L__BB206_5: // %atomicrmw.start19
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd28;
+; SM70-NEXT: min.f64 %rd20, %rd9, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM70-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
+; SM70-NEXT: @%p3 bra $L__BB206_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end18
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd17+24];
+; SM70-NEXT: $L__BB206_7: // %atomicrmw.start24
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd29;
+; SM70-NEXT: min.f64 %rd21, %rd10, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
+; SM70-NEXT: @%p4 bra $L__BB206_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end23
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd17+32];
+; SM70-NEXT: $L__BB206_9: // %atomicrmw.start38
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd30;
+; SM70-NEXT: min.f64 %rd22, %rd13, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
+; SM70-NEXT: @%p5 bra $L__BB206_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end37
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd17+40];
+; SM70-NEXT: $L__BB206_11: // %atomicrmw.start43
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd31;
+; SM70-NEXT: min.f64 %rd23, %rd14, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
+; SM70-NEXT: @%p6 bra $L__BB206_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end42
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd17+48];
+; SM70-NEXT: $L__BB206_13: // %atomicrmw.start53
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd24, %rd32, %rd11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM70-NEXT: mov.b64 %rd32, %rd15;
+; SM70-NEXT: @%p7 bra $L__BB206_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end52
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd17+56];
+; SM70-NEXT: $L__BB206_15: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd25, %rd33, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
+; SM70-NEXT: mov.b64 %rd33, %rd16;
+; SM70-NEXT: @%p8 bra $L__BB206_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end57
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM70-LABEL: fmax_acq_rel_v2f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [fmax_acq_rel_v2f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd5];
+; SM70-NEXT: $L__BB207_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd6, %rd8, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM70-NEXT: mov.b64 %rd8, %rd3;
+; SM70-NEXT: @%p1 bra $L__BB207_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd5+8];
+; SM70-NEXT: $L__BB207_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd7, %rd9, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
+; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
+; SM70-NEXT: mov.b64 %rd9, %rd4;
+; SM70-NEXT: @%p2 bra $L__BB207_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM70-LABEL: fmax_acq_rel_v4f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [fmax_acq_rel_v4f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd9];
+; SM70-NEXT: $L__BB208_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd14;
+; SM70-NEXT: max.f64 %rd10, %rd5, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
+; SM70-NEXT: @%p1 bra $L__BB208_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd9+8];
+; SM70-NEXT: $L__BB208_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd15;
+; SM70-NEXT: max.f64 %rd11, %rd6, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
+; SM70-NEXT: @%p2 bra $L__BB208_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd9+16];
+; SM70-NEXT: $L__BB208_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd12, %rd16, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM70-NEXT: mov.b64 %rd16, %rd7;
+; SM70-NEXT: @%p3 bra $L__BB208_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd9+24];
+; SM70-NEXT: $L__BB208_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd13, %rd17, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
+; SM70-NEXT: mov.b64 %rd17, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB208_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM70-LABEL: fmax_acq_rel_v8f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<34>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [fmax_acq_rel_v8f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmax_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmax_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd17];
+; SM70-NEXT: $L__BB209_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd26;
+; SM70-NEXT: max.f64 %rd18, %rd7, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM70-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
+; SM70-NEXT: @%p1 bra $L__BB209_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd17+8];
+; SM70-NEXT: $L__BB209_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd27;
+; SM70-NEXT: max.f64 %rd19, %rd8, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
+; SM70-NEXT: @%p2 bra $L__BB209_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd17+16];
+; SM70-NEXT: $L__BB209_5: // %atomicrmw.start19
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd28;
+; SM70-NEXT: max.f64 %rd20, %rd9, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM70-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
+; SM70-NEXT: @%p3 bra $L__BB209_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end18
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd17+24];
+; SM70-NEXT: $L__BB209_7: // %atomicrmw.start24
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd29;
+; SM70-NEXT: max.f64 %rd21, %rd10, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
+; SM70-NEXT: @%p4 bra $L__BB209_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end23
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd17+32];
+; SM70-NEXT: $L__BB209_9: // %atomicrmw.start38
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd30;
+; SM70-NEXT: max.f64 %rd22, %rd13, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
+; SM70-NEXT: @%p5 bra $L__BB209_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end37
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd17+40];
+; SM70-NEXT: $L__BB209_11: // %atomicrmw.start43
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd31;
+; SM70-NEXT: max.f64 %rd23, %rd14, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
+; SM70-NEXT: @%p6 bra $L__BB209_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end42
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd17+48];
+; SM70-NEXT: $L__BB209_13: // %atomicrmw.start53
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd24, %rd32, %rd11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM70-NEXT: mov.b64 %rd32, %rd15;
+; SM70-NEXT: @%p7 bra $L__BB209_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end52
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd17+56];
+; SM70-NEXT: $L__BB209_15: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd25, %rd33, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
+; SM70-NEXT: mov.b64 %rd33, %rd16;
+; SM70-NEXT: @%p8 bra $L__BB209_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end57
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM70-LABEL: fminimum_acq_rel_v2f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [fminimum_acq_rel_v2f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd5];
+; SM70-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM70-NEXT: $L__BB210_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
+; SM70-NEXT: min.f64 %rd6, %rd16, %rd1;
+; SM70-NEXT: selp.f64 %rd7, 0d7FF8000000000000, %rd6, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd16, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd8, %rd16, %rd7, %p2;
+; SM70-NEXT: selp.f64 %rd9, %rd1, %rd8, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd7, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd10, %rd9, %rd7, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
+; SM70-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
+; SM70-NEXT: mov.b64 %rd16, %rd3;
+; SM70-NEXT: @%p5 bra $L__BB210_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd5+8];
+; SM70-NEXT: setp.eq.b64 %p8, %rd2, -9223372036854775808;
+; SM70-NEXT: $L__BB210_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
+; SM70-NEXT: min.f64 %rd11, %rd17, %rd2;
+; SM70-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
+; SM70-NEXT: setp.eq.b64 %p7, %rd17, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd13, %rd17, %rd12, %p7;
+; SM70-NEXT: selp.f64 %rd14, %rd2, %rd13, %p8;
+; SM70-NEXT: setp.eq.f64 %p9, %rd12, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd15, %rd14, %rd12, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
+; SM70-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
+; SM70-NEXT: mov.b64 %rd17, %rd4;
+; SM70-NEXT: @%p10 bra $L__BB210_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM70-LABEL: fminimum_acq_rel_v4f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<21>;
+; SM70-NEXT: .reg .b64 %rd<34>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd9];
+; SM70-NEXT: setp.eq.b64 %p3, %rd3, -9223372036854775808;
+; SM70-NEXT: $L__BB211_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd30;
+; SM70-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
+; SM70-NEXT: min.f64 %rd10, %rd5, %rd3;
+; SM70-NEXT: selp.f64 %rd11, 0d7FF8000000000000, %rd10, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd5, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd12, %rd5, %rd11, %p2;
+; SM70-NEXT: selp.f64 %rd13, %rd3, %rd12, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd11, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
+; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
+; SM70-NEXT: @%p5 bra $L__BB211_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd9+8];
+; SM70-NEXT: setp.eq.b64 %p8, %rd4, -9223372036854775808;
+; SM70-NEXT: $L__BB211_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd31;
+; SM70-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
+; SM70-NEXT: min.f64 %rd15, %rd6, %rd4;
+; SM70-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
+; SM70-NEXT: setp.eq.b64 %p7, %rd6, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd17, %rd6, %rd16, %p7;
+; SM70-NEXT: selp.f64 %rd18, %rd4, %rd17, %p8;
+; SM70-NEXT: setp.eq.f64 %p9, %rd16, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
+; SM70-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
+; SM70-NEXT: @%p10 bra $L__BB211_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd9+16];
+; SM70-NEXT: setp.eq.b64 %p13, %rd1, -9223372036854775808;
+; SM70-NEXT: $L__BB211_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
+; SM70-NEXT: min.f64 %rd20, %rd32, %rd1;
+; SM70-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p11;
+; SM70-NEXT: setp.eq.b64 %p12, %rd32, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd22, %rd32, %rd21, %p12;
+; SM70-NEXT: selp.f64 %rd23, %rd1, %rd22, %p13;
+; SM70-NEXT: setp.eq.f64 %p14, %rd21, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd24, %rd23, %rd21, %p14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
+; SM70-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
+; SM70-NEXT: mov.b64 %rd32, %rd7;
+; SM70-NEXT: @%p15 bra $L__BB211_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd9+24];
+; SM70-NEXT: setp.eq.b64 %p18, %rd2, -9223372036854775808;
+; SM70-NEXT: $L__BB211_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
+; SM70-NEXT: min.f64 %rd25, %rd33, %rd2;
+; SM70-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p16;
+; SM70-NEXT: setp.eq.b64 %p17, %rd33, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd27, %rd33, %rd26, %p17;
+; SM70-NEXT: selp.f64 %rd28, %rd2, %rd27, %p18;
+; SM70-NEXT: setp.eq.f64 %p19, %rd26, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd29, %rd28, %rd26, %p19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
+; SM70-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
+; SM70-NEXT: mov.b64 %rd33, %rd8;
+; SM70-NEXT: @%p20 bra $L__BB211_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd30, %rd31};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM70-LABEL: fminimum_acq_rel_v8f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<41>;
+; SM70-NEXT: .reg .b64 %rd<66>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fminimum_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd58, [%rd17];
+; SM70-NEXT: setp.eq.b64 %p3, %rd5, -9223372036854775808;
+; SM70-NEXT: $L__BB212_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd58;
+; SM70-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
+; SM70-NEXT: min.f64 %rd18, %rd7, %rd5;
+; SM70-NEXT: selp.f64 %rd19, 0d7FF8000000000000, %rd18, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd7, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd20, %rd7, %rd19, %p2;
+; SM70-NEXT: selp.f64 %rd21, %rd5, %rd20, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd19, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
+; SM70-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
+; SM70-NEXT: @%p5 bra $L__BB212_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd59, [%rd17+8];
+; SM70-NEXT: setp.eq.b64 %p8, %rd6, -9223372036854775808;
+; SM70-NEXT: $L__BB212_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd59;
+; SM70-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
+; SM70-NEXT: min.f64 %rd23, %rd8, %rd6;
+; SM70-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
+; SM70-NEXT: setp.eq.b64 %p7, %rd8, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd25, %rd8, %rd24, %p7;
+; SM70-NEXT: selp.f64 %rd26, %rd6, %rd25, %p8;
+; SM70-NEXT: setp.eq.f64 %p9, %rd24, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
+; SM70-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
+; SM70-NEXT: @%p10 bra $L__BB212_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd60, [%rd17+16];
+; SM70-NEXT: setp.eq.b64 %p13, %rd3, -9223372036854775808;
+; SM70-NEXT: $L__BB212_5: // %atomicrmw.start19
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd60;
+; SM70-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
+; SM70-NEXT: min.f64 %rd28, %rd9, %rd3;
+; SM70-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p11;
+; SM70-NEXT: setp.eq.b64 %p12, %rd9, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd30, %rd9, %rd29, %p12;
+; SM70-NEXT: selp.f64 %rd31, %rd3, %rd30, %p13;
+; SM70-NEXT: setp.eq.f64 %p14, %rd29, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
+; SM70-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
+; SM70-NEXT: @%p15 bra $L__BB212_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end18
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd61, [%rd17+24];
+; SM70-NEXT: setp.eq.b64 %p18, %rd4, -9223372036854775808;
+; SM70-NEXT: $L__BB212_7: // %atomicrmw.start24
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd61;
+; SM70-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
+; SM70-NEXT: min.f64 %rd33, %rd10, %rd4;
+; SM70-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p16;
+; SM70-NEXT: setp.eq.b64 %p17, %rd10, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd35, %rd10, %rd34, %p17;
+; SM70-NEXT: selp.f64 %rd36, %rd4, %rd35, %p18;
+; SM70-NEXT: setp.eq.f64 %p19, %rd34, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
+; SM70-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
+; SM70-NEXT: @%p20 bra $L__BB212_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end23
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd62, [%rd17+32];
+; SM70-NEXT: setp.eq.b64 %p23, %rd1, -9223372036854775808;
+; SM70-NEXT: $L__BB212_9: // %atomicrmw.start38
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd62;
+; SM70-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
+; SM70-NEXT: min.f64 %rd38, %rd13, %rd1;
+; SM70-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p21;
+; SM70-NEXT: setp.eq.b64 %p22, %rd13, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd40, %rd13, %rd39, %p22;
+; SM70-NEXT: selp.f64 %rd41, %rd1, %rd40, %p23;
+; SM70-NEXT: setp.eq.f64 %p24, %rd39, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
+; SM70-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
+; SM70-NEXT: @%p25 bra $L__BB212_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end37
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd63, [%rd17+40];
+; SM70-NEXT: setp.eq.b64 %p28, %rd2, -9223372036854775808;
+; SM70-NEXT: $L__BB212_11: // %atomicrmw.start43
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd63;
+; SM70-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
+; SM70-NEXT: min.f64 %rd43, %rd14, %rd2;
+; SM70-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
+; SM70-NEXT: setp.eq.b64 %p27, %rd14, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd45, %rd14, %rd44, %p27;
+; SM70-NEXT: selp.f64 %rd46, %rd2, %rd45, %p28;
+; SM70-NEXT: setp.eq.f64 %p29, %rd44, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
+; SM70-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
+; SM70-NEXT: @%p30 bra $L__BB212_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end42
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd64, [%rd17+48];
+; SM70-NEXT: setp.eq.b64 %p33, %rd11, -9223372036854775808;
+; SM70-NEXT: $L__BB212_13: // %atomicrmw.start53
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
+; SM70-NEXT: min.f64 %rd48, %rd64, %rd11;
+; SM70-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p31;
+; SM70-NEXT: setp.eq.b64 %p32, %rd64, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd50, %rd64, %rd49, %p32;
+; SM70-NEXT: selp.f64 %rd51, %rd11, %rd50, %p33;
+; SM70-NEXT: setp.eq.f64 %p34, %rd49, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd52, %rd51, %rd49, %p34;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
+; SM70-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
+; SM70-NEXT: mov.b64 %rd64, %rd15;
+; SM70-NEXT: @%p35 bra $L__BB212_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end52
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd65, [%rd17+56];
+; SM70-NEXT: setp.eq.b64 %p38, %rd12, -9223372036854775808;
+; SM70-NEXT: $L__BB212_15: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
+; SM70-NEXT: min.f64 %rd53, %rd65, %rd12;
+; SM70-NEXT: selp.f64 %rd54, 0d7FF8000000000000, %rd53, %p36;
+; SM70-NEXT: setp.eq.b64 %p37, %rd65, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd55, %rd65, %rd54, %p37;
+; SM70-NEXT: selp.f64 %rd56, %rd12, %rd55, %p38;
+; SM70-NEXT: setp.eq.f64 %p39, %rd54, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd57, %rd56, %rd54, %p39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
+; SM70-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
+; SM70-NEXT: mov.b64 %rd65, %rd16;
+; SM70-NEXT: @%p40 bra $L__BB212_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end57
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd62, %rd63};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v2f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [fmaximum_acq_rel_v2f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd5];
+; SM70-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM70-NEXT: $L__BB213_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
+; SM70-NEXT: max.f64 %rd6, %rd16, %rd1;
+; SM70-NEXT: selp.f64 %rd7, 0d7FF8000000000000, %rd6, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd16, 0;
+; SM70-NEXT: selp.f64 %rd8, %rd16, %rd7, %p2;
+; SM70-NEXT: selp.f64 %rd9, %rd1, %rd8, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd7, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd10, %rd9, %rd7, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
+; SM70-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
+; SM70-NEXT: mov.b64 %rd16, %rd3;
+; SM70-NEXT: @%p5 bra $L__BB213_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd5+8];
+; SM70-NEXT: setp.eq.b64 %p8, %rd2, 0;
+; SM70-NEXT: $L__BB213_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
+; SM70-NEXT: max.f64 %rd11, %rd17, %rd2;
+; SM70-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
+; SM70-NEXT: setp.eq.b64 %p7, %rd17, 0;
+; SM70-NEXT: selp.f64 %rd13, %rd17, %rd12, %p7;
+; SM70-NEXT: selp.f64 %rd14, %rd2, %rd13, %p8;
+; SM70-NEXT: setp.eq.f64 %p9, %rd12, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd15, %rd14, %rd12, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
+; SM70-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
+; SM70-NEXT: mov.b64 %rd17, %rd4;
+; SM70-NEXT: @%p10 bra $L__BB213_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v4f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<21>;
+; SM70-NEXT: .reg .b64 %rd<34>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd9];
+; SM70-NEXT: setp.eq.b64 %p3, %rd3, 0;
+; SM70-NEXT: $L__BB214_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd30;
+; SM70-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
+; SM70-NEXT: max.f64 %rd10, %rd5, %rd3;
+; SM70-NEXT: selp.f64 %rd11, 0d7FF8000000000000, %rd10, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd5, 0;
+; SM70-NEXT: selp.f64 %rd12, %rd5, %rd11, %p2;
+; SM70-NEXT: selp.f64 %rd13, %rd3, %rd12, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd11, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
+; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
+; SM70-NEXT: @%p5 bra $L__BB214_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd9+8];
+; SM70-NEXT: setp.eq.b64 %p8, %rd4, 0;
+; SM70-NEXT: $L__BB214_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd31;
+; SM70-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
+; SM70-NEXT: max.f64 %rd15, %rd6, %rd4;
+; SM70-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
+; SM70-NEXT: setp.eq.b64 %p7, %rd6, 0;
+; SM70-NEXT: selp.f64 %rd17, %rd6, %rd16, %p7;
+; SM70-NEXT: selp.f64 %rd18, %rd4, %rd17, %p8;
+; SM70-NEXT: setp.eq.f64 %p9, %rd16, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
+; SM70-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
+; SM70-NEXT: @%p10 bra $L__BB214_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd9+16];
+; SM70-NEXT: setp.eq.b64 %p13, %rd1, 0;
+; SM70-NEXT: $L__BB214_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
+; SM70-NEXT: max.f64 %rd20, %rd32, %rd1;
+; SM70-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p11;
+; SM70-NEXT: setp.eq.b64 %p12, %rd32, 0;
+; SM70-NEXT: selp.f64 %rd22, %rd32, %rd21, %p12;
+; SM70-NEXT: selp.f64 %rd23, %rd1, %rd22, %p13;
+; SM70-NEXT: setp.eq.f64 %p14, %rd21, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd24, %rd23, %rd21, %p14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
+; SM70-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
+; SM70-NEXT: mov.b64 %rd32, %rd7;
+; SM70-NEXT: @%p15 bra $L__BB214_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd9+24];
+; SM70-NEXT: setp.eq.b64 %p18, %rd2, 0;
+; SM70-NEXT: $L__BB214_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
+; SM70-NEXT: max.f64 %rd25, %rd33, %rd2;
+; SM70-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p16;
+; SM70-NEXT: setp.eq.b64 %p17, %rd33, 0;
+; SM70-NEXT: selp.f64 %rd27, %rd33, %rd26, %p17;
+; SM70-NEXT: selp.f64 %rd28, %rd2, %rd27, %p18;
+; SM70-NEXT: setp.eq.f64 %p19, %rd26, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd29, %rd28, %rd26, %p19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
+; SM70-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
+; SM70-NEXT: mov.b64 %rd33, %rd8;
+; SM70-NEXT: @%p20 bra $L__BB214_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd30, %rd31};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v8f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<41>;
+; SM70-NEXT: .reg .b64 %rd<66>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmaximum_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd58, [%rd17];
+; SM70-NEXT: setp.eq.b64 %p3, %rd5, 0;
+; SM70-NEXT: $L__BB215_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd58;
+; SM70-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
+; SM70-NEXT: max.f64 %rd18, %rd7, %rd5;
+; SM70-NEXT: selp.f64 %rd19, 0d7FF8000000000000, %rd18, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd7, 0;
+; SM70-NEXT: selp.f64 %rd20, %rd7, %rd19, %p2;
+; SM70-NEXT: selp.f64 %rd21, %rd5, %rd20, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd19, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
+; SM70-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
+; SM70-NEXT: @%p5 bra $L__BB215_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd59, [%rd17+8];
+; SM70-NEXT: setp.eq.b64 %p8, %rd6, 0;
+; SM70-NEXT: $L__BB215_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd59;
+; SM70-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
+; SM70-NEXT: max.f64 %rd23, %rd8, %rd6;
+; SM70-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
+; SM70-NEXT: setp.eq.b64 %p7, %rd8, 0;
+; SM70-NEXT: selp.f64 %rd25, %rd8, %rd24, %p7;
+; SM70-NEXT: selp.f64 %rd26, %rd6, %rd25, %p8;
+; SM70-NEXT: setp.eq.f64 %p9, %rd24, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
+; SM70-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
+; SM70-NEXT: @%p10 bra $L__BB215_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd60, [%rd17+16];
+; SM70-NEXT: setp.eq.b64 %p13, %rd3, 0;
+; SM70-NEXT: $L__BB215_5: // %atomicrmw.start19
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd60;
+; SM70-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
+; SM70-NEXT: max.f64 %rd28, %rd9, %rd3;
+; SM70-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p11;
+; SM70-NEXT: setp.eq.b64 %p12, %rd9, 0;
+; SM70-NEXT: selp.f64 %rd30, %rd9, %rd29, %p12;
+; SM70-NEXT: selp.f64 %rd31, %rd3, %rd30, %p13;
+; SM70-NEXT: setp.eq.f64 %p14, %rd29, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
+; SM70-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
+; SM70-NEXT: @%p15 bra $L__BB215_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end18
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd61, [%rd17+24];
+; SM70-NEXT: setp.eq.b64 %p18, %rd4, 0;
+; SM70-NEXT: $L__BB215_7: // %atomicrmw.start24
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd61;
+; SM70-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
+; SM70-NEXT: max.f64 %rd33, %rd10, %rd4;
+; SM70-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p16;
+; SM70-NEXT: setp.eq.b64 %p17, %rd10, 0;
+; SM70-NEXT: selp.f64 %rd35, %rd10, %rd34, %p17;
+; SM70-NEXT: selp.f64 %rd36, %rd4, %rd35, %p18;
+; SM70-NEXT: setp.eq.f64 %p19, %rd34, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
+; SM70-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
+; SM70-NEXT: @%p20 bra $L__BB215_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end23
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd62, [%rd17+32];
+; SM70-NEXT: setp.eq.b64 %p23, %rd1, 0;
+; SM70-NEXT: $L__BB215_9: // %atomicrmw.start38
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd62;
+; SM70-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
+; SM70-NEXT: max.f64 %rd38, %rd13, %rd1;
+; SM70-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p21;
+; SM70-NEXT: setp.eq.b64 %p22, %rd13, 0;
+; SM70-NEXT: selp.f64 %rd40, %rd13, %rd39, %p22;
+; SM70-NEXT: selp.f64 %rd41, %rd1, %rd40, %p23;
+; SM70-NEXT: setp.eq.f64 %p24, %rd39, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
+; SM70-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
+; SM70-NEXT: @%p25 bra $L__BB215_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end37
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd63, [%rd17+40];
+; SM70-NEXT: setp.eq.b64 %p28, %rd2, 0;
+; SM70-NEXT: $L__BB215_11: // %atomicrmw.start43
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd63;
+; SM70-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
+; SM70-NEXT: max.f64 %rd43, %rd14, %rd2;
+; SM70-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
+; SM70-NEXT: setp.eq.b64 %p27, %rd14, 0;
+; SM70-NEXT: selp.f64 %rd45, %rd14, %rd44, %p27;
+; SM70-NEXT: selp.f64 %rd46, %rd2, %rd45, %p28;
+; SM70-NEXT: setp.eq.f64 %p29, %rd44, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
+; SM70-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
+; SM70-NEXT: @%p30 bra $L__BB215_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end42
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd64, [%rd17+48];
+; SM70-NEXT: setp.eq.b64 %p33, %rd11, 0;
+; SM70-NEXT: $L__BB215_13: // %atomicrmw.start53
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
+; SM70-NEXT: max.f64 %rd48, %rd64, %rd11;
+; SM70-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p31;
+; SM70-NEXT: setp.eq.b64 %p32, %rd64, 0;
+; SM70-NEXT: selp.f64 %rd50, %rd64, %rd49, %p32;
+; SM70-NEXT: selp.f64 %rd51, %rd11, %rd50, %p33;
+; SM70-NEXT: setp.eq.f64 %p34, %rd49, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd52, %rd51, %rd49, %p34;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
+; SM70-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
+; SM70-NEXT: mov.b64 %rd64, %rd15;
+; SM70-NEXT: @%p35 bra $L__BB215_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end52
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd65, [%rd17+56];
+; SM70-NEXT: setp.eq.b64 %p38, %rd12, 0;
+; SM70-NEXT: $L__BB215_15: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
+; SM70-NEXT: max.f64 %rd53, %rd65, %rd12;
+; SM70-NEXT: selp.f64 %rd54, 0d7FF8000000000000, %rd53, %p36;
+; SM70-NEXT: setp.eq.b64 %p37, %rd65, 0;
+; SM70-NEXT: selp.f64 %rd55, %rd65, %rd54, %p37;
+; SM70-NEXT: selp.f64 %rd56, %rd12, %rd55, %p38;
+; SM70-NEXT: setp.eq.f64 %p39, %rd54, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd57, %rd56, %rd54, %p39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
+; SM70-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
+; SM70-NEXT: mov.b64 %rd65, %rd16;
+; SM70-NEXT: @%p40 bra $L__BB215_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end57
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd62, %rd63};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM70-LABEL: fadd_acq_rel_v2f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM70-LABEL: fadd_acq_rel_v4f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs5, [%rd1], %rs1;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs2;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs3;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM70-LABEL: fadd_acq_rel_v8f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f16_global_cta_param_1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs9, [%rd1], %rs7;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+2], %rs8;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+4], %rs5;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs6;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs13, [%rd1+8], %rs3;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+10], %rs4;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs1;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: mov.b32 %r5, {%rs9, %rs10};
+; SM70-NEXT: mov.b32 %r6, {%rs11, %rs12};
+; SM70-NEXT: mov.b32 %r7, {%rs13, %rs14};
+; SM70-NEXT: mov.b32 %r8, {%rs15, %rs16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM70-LABEL: fsub_acq_rel_v2f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB219_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f16x2 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB219_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM70-LABEL: fsub_acq_rel_v4f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB220_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f16x2 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: sub.rn.f16x2 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB220_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM70-LABEL: fsub_acq_rel_v8f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB221_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f16x2 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: sub.rn.f16x2 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB221_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB221_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f16x2 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: sub.rn.f16x2 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB221_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM70-LABEL: fmin_acq_rel_v2f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<11>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r10, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM70-NEXT: $L__BB222_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r10;
+; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM70-NEXT: min.f32 %r5, %r4, %r3;
+; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM70-NEXT: min.f32 %r8, %r7, %r6;
+; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM70-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r10;
+; SM70-NEXT: mov.b32 %r10, %r1;
+; SM70-NEXT: @%p1 bra $L__BB222_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM70-LABEL: fmin_acq_rel_v4f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: cvt.f32.f16 %r10, %rs8;
+; SM70-NEXT: cvt.f32.f16 %r13, %rs7;
+; SM70-NEXT: $L__BB223_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r17;
+; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM70-NEXT: min.f32 %r5, %r4, %r3;
+; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM70-NEXT: min.f32 %r8, %r7, %r6;
+; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM70-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r18;
+; SM70-NEXT: cvt.f32.f16 %r11, %rs10;
+; SM70-NEXT: min.f32 %r12, %r11, %r10;
+; SM70-NEXT: cvt.rn.f16.f32 %rs11, %r12;
+; SM70-NEXT: cvt.f32.f16 %r14, %rs9;
+; SM70-NEXT: min.f32 %r15, %r14, %r13;
+; SM70-NEXT: cvt.rn.f16.f32 %rs12, %r15;
+; SM70-NEXT: mov.b32 %r16, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r16;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB223_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r17, %r18};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM70-LABEL: fmin_acq_rel_v8f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<37>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.f32.f16 %r5, %rs2;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: cvt.f32.f16 %r12, %rs8;
+; SM70-NEXT: $L__BB224_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r33;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs4;
+; SM70-NEXT: min.f32 %r7, %r6, %r5;
+; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r7;
+; SM70-NEXT: cvt.f32.f16 %r8, %rs1;
+; SM70-NEXT: cvt.f32.f16 %r9, %rs3;
+; SM70-NEXT: min.f32 %r10, %r9, %r8;
+; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r10;
+; SM70-NEXT: mov.b32 %r11, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r34;
+; SM70-NEXT: cvt.f32.f16 %r13, %rs10;
+; SM70-NEXT: min.f32 %r14, %r13, %r12;
+; SM70-NEXT: cvt.rn.f16.f32 %rs11, %r14;
+; SM70-NEXT: cvt.f32.f16 %r15, %rs7;
+; SM70-NEXT: cvt.f32.f16 %r16, %rs9;
+; SM70-NEXT: min.f32 %r17, %r16, %r15;
+; SM70-NEXT: cvt.rn.f16.f32 %rs12, %r17;
+; SM70-NEXT: mov.b32 %r18, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB224_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r35, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: cvt.f32.f16 %r19, %rs14;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: cvt.f32.f16 %r26, %rs20;
+; SM70-NEXT: $L__BB224_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r35;
+; SM70-NEXT: cvt.f32.f16 %r20, %rs16;
+; SM70-NEXT: min.f32 %r21, %r20, %r19;
+; SM70-NEXT: cvt.rn.f16.f32 %rs17, %r21;
+; SM70-NEXT: cvt.f32.f16 %r22, %rs13;
+; SM70-NEXT: cvt.f32.f16 %r23, %rs15;
+; SM70-NEXT: min.f32 %r24, %r23, %r22;
+; SM70-NEXT: cvt.rn.f16.f32 %rs18, %r24;
+; SM70-NEXT: mov.b32 %r25, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r36;
+; SM70-NEXT: cvt.f32.f16 %r27, %rs22;
+; SM70-NEXT: min.f32 %r28, %r27, %r26;
+; SM70-NEXT: cvt.rn.f16.f32 %rs23, %r28;
+; SM70-NEXT: cvt.f32.f16 %r29, %rs19;
+; SM70-NEXT: cvt.f32.f16 %r30, %rs21;
+; SM70-NEXT: min.f32 %r31, %r30, %r29;
+; SM70-NEXT: cvt.rn.f16.f32 %rs24, %r31;
+; SM70-NEXT: mov.b32 %r32, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r32;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r35;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r36;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r35, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB224_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM70-LABEL: fmax_acq_rel_v2f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<11>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r10, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM70-NEXT: $L__BB225_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r10;
+; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM70-NEXT: max.f32 %r5, %r4, %r3;
+; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM70-NEXT: max.f32 %r8, %r7, %r6;
+; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM70-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r10;
+; SM70-NEXT: mov.b32 %r10, %r1;
+; SM70-NEXT: @%p1 bra $L__BB225_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM70-LABEL: fmax_acq_rel_v4f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: cvt.f32.f16 %r10, %rs8;
+; SM70-NEXT: cvt.f32.f16 %r13, %rs7;
+; SM70-NEXT: $L__BB226_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r17;
+; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM70-NEXT: max.f32 %r5, %r4, %r3;
+; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM70-NEXT: max.f32 %r8, %r7, %r6;
+; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM70-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r18;
+; SM70-NEXT: cvt.f32.f16 %r11, %rs10;
+; SM70-NEXT: max.f32 %r12, %r11, %r10;
+; SM70-NEXT: cvt.rn.f16.f32 %rs11, %r12;
+; SM70-NEXT: cvt.f32.f16 %r14, %rs9;
+; SM70-NEXT: max.f32 %r15, %r14, %r13;
+; SM70-NEXT: cvt.rn.f16.f32 %rs12, %r15;
+; SM70-NEXT: mov.b32 %r16, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r16;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB226_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r17, %r18};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM70-LABEL: fmax_acq_rel_v8f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<37>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.f32.f16 %r5, %rs2;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: cvt.f32.f16 %r12, %rs8;
+; SM70-NEXT: $L__BB227_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r33;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs4;
+; SM70-NEXT: max.f32 %r7, %r6, %r5;
+; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r7;
+; SM70-NEXT: cvt.f32.f16 %r8, %rs1;
+; SM70-NEXT: cvt.f32.f16 %r9, %rs3;
+; SM70-NEXT: max.f32 %r10, %r9, %r8;
+; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r10;
+; SM70-NEXT: mov.b32 %r11, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r34;
+; SM70-NEXT: cvt.f32.f16 %r13, %rs10;
+; SM70-NEXT: max.f32 %r14, %r13, %r12;
+; SM70-NEXT: cvt.rn.f16.f32 %rs11, %r14;
+; SM70-NEXT: cvt.f32.f16 %r15, %rs7;
+; SM70-NEXT: cvt.f32.f16 %r16, %rs9;
+; SM70-NEXT: max.f32 %r17, %r16, %r15;
+; SM70-NEXT: cvt.rn.f16.f32 %rs12, %r17;
+; SM70-NEXT: mov.b32 %r18, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB227_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r35, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: cvt.f32.f16 %r19, %rs14;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: cvt.f32.f16 %r26, %rs20;
+; SM70-NEXT: $L__BB227_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r35;
+; SM70-NEXT: cvt.f32.f16 %r20, %rs16;
+; SM70-NEXT: max.f32 %r21, %r20, %r19;
+; SM70-NEXT: cvt.rn.f16.f32 %rs17, %r21;
+; SM70-NEXT: cvt.f32.f16 %r22, %rs13;
+; SM70-NEXT: cvt.f32.f16 %r23, %rs15;
+; SM70-NEXT: max.f32 %r24, %r23, %r22;
+; SM70-NEXT: cvt.rn.f16.f32 %rs18, %r24;
+; SM70-NEXT: mov.b32 %r25, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r36;
+; SM70-NEXT: cvt.f32.f16 %r27, %rs22;
+; SM70-NEXT: max.f32 %r28, %r27, %r26;
+; SM70-NEXT: cvt.rn.f16.f32 %rs23, %r28;
+; SM70-NEXT: cvt.f32.f16 %r29, %rs19;
+; SM70-NEXT: cvt.f32.f16 %r30, %rs21;
+; SM70-NEXT: max.f32 %r31, %r30, %r29;
+; SM70-NEXT: cvt.rn.f16.f32 %rs24, %r31;
+; SM70-NEXT: mov.b32 %r32, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r32;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r35;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r36;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r35, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB227_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM70-LABEL: fminimum_acq_rel_v2f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<12>;
+; SM70-NEXT: .reg .b16 %rs<16>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM70-NEXT: $L__BB228_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: mov.b16 %rs9, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM70-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p11, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p11 bra $L__BB228_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM70-LABEL: fminimum_acq_rel_v4f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<22>;
+; SM70-NEXT: .reg .b16 %rs<30>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM70-NEXT: setp.eq.b16 %p14, %rs17, -32768;
+; SM70-NEXT: setp.eq.b16 %p19, %rs16, -32768;
+; SM70-NEXT: $L__BB229_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: mov.b16 %rs9, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM70-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: mov.b32 {%rs18, %rs19}, %r6;
+; SM70-NEXT: setp.lt.f16 %p11, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM70-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs19, -32768;
+; SM70-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM70-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM70-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM70-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM70-NEXT: setp.lt.f16 %p16, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM70-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs18, -32768;
+; SM70-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM70-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM70-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM70-NEXT: mov.b32 %r4, {%rs29, %rs24};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB229_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM70-LABEL: fminimum_acq_rel_v8f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<43>;
+; SM70-NEXT: .reg .b16 %rs<59>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM70-NEXT: setp.eq.b16 %p14, %rs17, -32768;
+; SM70-NEXT: setp.eq.b16 %p19, %rs16, -32768;
+; SM70-NEXT: $L__BB230_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: mov.b16 %rs9, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM70-NEXT: mov.b32 %r5, {%rs15, %rs10};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: mov.b32 {%rs18, %rs19}, %r10;
+; SM70-NEXT: setp.lt.f16 %p11, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM70-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs19, -32768;
+; SM70-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM70-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM70-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM70-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM70-NEXT: setp.lt.f16 %p16, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM70-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs18, -32768;
+; SM70-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM70-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM70-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM70-NEXT: mov.b32 %r6, {%rs29, %rs24};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB230_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB230_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs30, %rs31}, %r3;
+; SM70-NEXT: mov.b32 {%rs32, %rs33}, %r11;
+; SM70-NEXT: setp.lt.f16 %p22, %rs33, %rs31;
+; SM70-NEXT: selp.b16 %rs34, %rs33, %rs31, %p22;
+; SM70-NEXT: setp.nan.f16 %p23, %rs33, %rs31;
+; SM70-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p23;
+; SM70-NEXT: setp.eq.b16 %p24, %rs33, -32768;
+; SM70-NEXT: selp.b16 %rs36, %rs33, %rs35, %p24;
+; SM70-NEXT: setp.eq.b16 %p25, %rs31, -32768;
+; SM70-NEXT: selp.b16 %rs37, %rs31, %rs36, %p25;
+; SM70-NEXT: mov.b16 %rs38, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p26, %rs35, %rs38;
+; SM70-NEXT: selp.b16 %rs39, %rs37, %rs35, %p26;
+; SM70-NEXT: setp.lt.f16 %p27, %rs32, %rs30;
+; SM70-NEXT: selp.b16 %rs40, %rs32, %rs30, %p27;
+; SM70-NEXT: setp.nan.f16 %p28, %rs32, %rs30;
+; SM70-NEXT: selp.b16 %rs41, 0x7E00, %rs40, %p28;
+; SM70-NEXT: setp.eq.b16 %p29, %rs32, -32768;
+; SM70-NEXT: selp.b16 %rs42, %rs32, %rs41, %p29;
+; SM70-NEXT: setp.eq.b16 %p30, %rs30, -32768;
+; SM70-NEXT: selp.b16 %rs43, %rs30, %rs42, %p30;
+; SM70-NEXT: setp.eq.f16 %p31, %rs41, %rs38;
+; SM70-NEXT: selp.b16 %rs44, %rs43, %rs41, %p31;
+; SM70-NEXT: mov.b32 %r7, {%rs44, %rs39};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: mov.b32 {%rs45, %rs46}, %r4;
+; SM70-NEXT: mov.b32 {%rs47, %rs48}, %r12;
+; SM70-NEXT: setp.lt.f16 %p32, %rs48, %rs46;
+; SM70-NEXT: selp.b16 %rs49, %rs48, %rs46, %p32;
+; SM70-NEXT: setp.nan.f16 %p33, %rs48, %rs46;
+; SM70-NEXT: selp.b16 %rs50, 0x7E00, %rs49, %p33;
+; SM70-NEXT: setp.eq.b16 %p34, %rs48, -32768;
+; SM70-NEXT: selp.b16 %rs51, %rs48, %rs50, %p34;
+; SM70-NEXT: setp.eq.b16 %p35, %rs46, -32768;
+; SM70-NEXT: selp.b16 %rs52, %rs46, %rs51, %p35;
+; SM70-NEXT: setp.eq.f16 %p36, %rs50, %rs38;
+; SM70-NEXT: selp.b16 %rs53, %rs52, %rs50, %p36;
+; SM70-NEXT: setp.lt.f16 %p37, %rs47, %rs45;
+; SM70-NEXT: selp.b16 %rs54, %rs47, %rs45, %p37;
+; SM70-NEXT: setp.nan.f16 %p38, %rs47, %rs45;
+; SM70-NEXT: selp.b16 %rs55, 0x7E00, %rs54, %p38;
+; SM70-NEXT: setp.eq.b16 %p39, %rs47, -32768;
+; SM70-NEXT: selp.b16 %rs56, %rs47, %rs55, %p39;
+; SM70-NEXT: setp.eq.b16 %p40, %rs45, -32768;
+; SM70-NEXT: selp.b16 %rs57, %rs45, %rs56, %p40;
+; SM70-NEXT: setp.eq.f16 %p41, %rs55, %rs38;
+; SM70-NEXT: selp.b16 %rs58, %rs57, %rs55, %p41;
+; SM70-NEXT: mov.b32 %r8, {%rs58, %rs53};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p42 bra $L__BB230_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v2f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<12>;
+; SM70-NEXT: .reg .b16 %rs<16>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM70-NEXT: $L__BB231_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: mov.b16 %rs9, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM70-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p11, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p11 bra $L__BB231_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v4f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<22>;
+; SM70-NEXT: .reg .b16 %rs<30>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM70-NEXT: setp.eq.b16 %p14, %rs17, 0;
+; SM70-NEXT: setp.eq.b16 %p19, %rs16, 0;
+; SM70-NEXT: $L__BB232_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: mov.b16 %rs9, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM70-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: mov.b32 {%rs18, %rs19}, %r6;
+; SM70-NEXT: setp.gt.f16 %p11, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM70-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs19, 0;
+; SM70-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM70-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM70-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM70-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM70-NEXT: setp.gt.f16 %p16, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM70-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs18, 0;
+; SM70-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM70-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM70-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM70-NEXT: mov.b32 %r4, {%rs29, %rs24};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB232_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v8f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<43>;
+; SM70-NEXT: .reg .b16 %rs<59>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM70-NEXT: setp.eq.b16 %p14, %rs17, 0;
+; SM70-NEXT: setp.eq.b16 %p19, %rs16, 0;
+; SM70-NEXT: $L__BB233_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: mov.b16 %rs9, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM70-NEXT: mov.b32 %r5, {%rs15, %rs10};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: mov.b32 {%rs18, %rs19}, %r10;
+; SM70-NEXT: setp.gt.f16 %p11, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM70-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs19, 0;
+; SM70-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM70-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM70-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM70-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM70-NEXT: setp.gt.f16 %p16, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM70-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs18, 0;
+; SM70-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM70-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM70-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM70-NEXT: mov.b32 %r6, {%rs29, %rs24};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB233_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB233_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs30, %rs31}, %r3;
+; SM70-NEXT: mov.b32 {%rs32, %rs33}, %r11;
+; SM70-NEXT: setp.gt.f16 %p22, %rs33, %rs31;
+; SM70-NEXT: selp.b16 %rs34, %rs33, %rs31, %p22;
+; SM70-NEXT: setp.nan.f16 %p23, %rs33, %rs31;
+; SM70-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p23;
+; SM70-NEXT: setp.eq.b16 %p24, %rs33, 0;
+; SM70-NEXT: selp.b16 %rs36, %rs33, %rs35, %p24;
+; SM70-NEXT: setp.eq.b16 %p25, %rs31, 0;
+; SM70-NEXT: selp.b16 %rs37, %rs31, %rs36, %p25;
+; SM70-NEXT: mov.b16 %rs38, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p26, %rs35, %rs38;
+; SM70-NEXT: selp.b16 %rs39, %rs37, %rs35, %p26;
+; SM70-NEXT: setp.gt.f16 %p27, %rs32, %rs30;
+; SM70-NEXT: selp.b16 %rs40, %rs32, %rs30, %p27;
+; SM70-NEXT: setp.nan.f16 %p28, %rs32, %rs30;
+; SM70-NEXT: selp.b16 %rs41, 0x7E00, %rs40, %p28;
+; SM70-NEXT: setp.eq.b16 %p29, %rs32, 0;
+; SM70-NEXT: selp.b16 %rs42, %rs32, %rs41, %p29;
+; SM70-NEXT: setp.eq.b16 %p30, %rs30, 0;
+; SM70-NEXT: selp.b16 %rs43, %rs30, %rs42, %p30;
+; SM70-NEXT: setp.eq.f16 %p31, %rs41, %rs38;
+; SM70-NEXT: selp.b16 %rs44, %rs43, %rs41, %p31;
+; SM70-NEXT: mov.b32 %r7, {%rs44, %rs39};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: mov.b32 {%rs45, %rs46}, %r4;
+; SM70-NEXT: mov.b32 {%rs47, %rs48}, %r12;
+; SM70-NEXT: setp.gt.f16 %p32, %rs48, %rs46;
+; SM70-NEXT: selp.b16 %rs49, %rs48, %rs46, %p32;
+; SM70-NEXT: setp.nan.f16 %p33, %rs48, %rs46;
+; SM70-NEXT: selp.b16 %rs50, 0x7E00, %rs49, %p33;
+; SM70-NEXT: setp.eq.b16 %p34, %rs48, 0;
+; SM70-NEXT: selp.b16 %rs51, %rs48, %rs50, %p34;
+; SM70-NEXT: setp.eq.b16 %p35, %rs46, 0;
+; SM70-NEXT: selp.b16 %rs52, %rs46, %rs51, %p35;
+; SM70-NEXT: setp.eq.f16 %p36, %rs50, %rs38;
+; SM70-NEXT: selp.b16 %rs53, %rs52, %rs50, %p36;
+; SM70-NEXT: setp.gt.f16 %p37, %rs47, %rs45;
+; SM70-NEXT: selp.b16 %rs54, %rs47, %rs45, %p37;
+; SM70-NEXT: setp.nan.f16 %p38, %rs47, %rs45;
+; SM70-NEXT: selp.b16 %rs55, 0x7E00, %rs54, %p38;
+; SM70-NEXT: setp.eq.b16 %p39, %rs47, 0;
+; SM70-NEXT: selp.b16 %rs56, %rs47, %rs55, %p39;
+; SM70-NEXT: setp.eq.b16 %p40, %rs45, 0;
+; SM70-NEXT: selp.b16 %rs57, %rs45, %rs56, %p40;
+; SM70-NEXT: setp.eq.f16 %p41, %rs55, %rs38;
+; SM70-NEXT: selp.b16 %rs58, %rs57, %rs55, %p41;
+; SM70-NEXT: mov.b32 %r8, {%rs58, %rs53};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p42 bra $L__BB233_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM70-LABEL: fadd_acq_rel_v2bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fadd_acq_rel_v2bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: $L__BB234_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: add.rn.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: add.rn.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM70-NEXT: mov.b32 %r24, %r1;
+; SM70-NEXT: @%p3 bra $L__BB234_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM70-LABEL: fadd_acq_rel_v4bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<47>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-NEXT: $L__BB235_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: add.rn.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: add.rn.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: add.rn.f32 %r28, %r27, %r25;
+; SM70-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM70-NEXT: add.s32 %r30, %r29, %r28;
+; SM70-NEXT: add.s32 %r31, %r30, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM70-NEXT: or.b32 %r32, %r28, 4194304;
+; SM70-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: add.rn.f32 %r38, %r37, %r35;
+; SM70-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM70-NEXT: add.s32 %r40, %r39, %r38;
+; SM70-NEXT: add.s32 %r41, %r40, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM70-NEXT: or.b32 %r42, %r38, 4194304;
+; SM70-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM70-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB235_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM70-LABEL: fadd_acq_rel_v8bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<93>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: $L__BB236_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: add.rn.f32 %r9, %r8, %r6;
+; SM70-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM70-NEXT: add.s32 %r11, %r10, %r9;
+; SM70-NEXT: add.s32 %r12, %r11, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM70-NEXT: or.b32 %r13, %r9, 4194304;
+; SM70-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM70-NEXT: shl.b32 %r18, %r17, 16;
+; SM70-NEXT: add.rn.f32 %r19, %r18, %r16;
+; SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM70-NEXT: add.s32 %r21, %r20, %r19;
+; SM70-NEXT: add.s32 %r22, %r21, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM70-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM70-NEXT: shl.b32 %r29, %r28, 16;
+; SM70-NEXT: add.rn.f32 %r30, %r29, %r27;
+; SM70-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM70-NEXT: add.s32 %r32, %r31, %r30;
+; SM70-NEXT: add.s32 %r33, %r32, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM70-NEXT: or.b32 %r34, %r30, 4194304;
+; SM70-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM70-NEXT: shl.b32 %r39, %r38, 16;
+; SM70-NEXT: add.rn.f32 %r40, %r39, %r37;
+; SM70-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM70-NEXT: add.s32 %r42, %r41, %r40;
+; SM70-NEXT: add.s32 %r43, %r42, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM70-NEXT: or.b32 %r44, %r40, 4194304;
+; SM70-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM70-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB236_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM70-NEXT: $L__BB236_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM70-NEXT: shl.b32 %r48, %r47, 16;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM70-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM70-NEXT: shl.b32 %r50, %r49, 16;
+; SM70-NEXT: add.rn.f32 %r51, %r50, %r48;
+; SM70-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM70-NEXT: add.s32 %r53, %r52, %r51;
+; SM70-NEXT: add.s32 %r54, %r53, 32767;
+; SM70-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM70-NEXT: or.b32 %r55, %r51, 4194304;
+; SM70-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM70-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM70-NEXT: shl.b32 %r58, %r57, 16;
+; SM70-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM70-NEXT: shl.b32 %r60, %r59, 16;
+; SM70-NEXT: add.rn.f32 %r61, %r60, %r58;
+; SM70-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM70-NEXT: add.s32 %r63, %r62, %r61;
+; SM70-NEXT: add.s32 %r64, %r63, 32767;
+; SM70-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM70-NEXT: or.b32 %r65, %r61, 4194304;
+; SM70-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM70-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM70-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM70-NEXT: shl.b32 %r69, %r68, 16;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM70-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM70-NEXT: shl.b32 %r71, %r70, 16;
+; SM70-NEXT: add.rn.f32 %r72, %r71, %r69;
+; SM70-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM70-NEXT: add.s32 %r74, %r73, %r72;
+; SM70-NEXT: add.s32 %r75, %r74, 32767;
+; SM70-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM70-NEXT: or.b32 %r76, %r72, 4194304;
+; SM70-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM70-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM70-NEXT: shl.b32 %r79, %r78, 16;
+; SM70-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM70-NEXT: shl.b32 %r81, %r80, 16;
+; SM70-NEXT: add.rn.f32 %r82, %r81, %r79;
+; SM70-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM70-NEXT: add.s32 %r84, %r83, %r82;
+; SM70-NEXT: add.s32 %r85, %r84, 32767;
+; SM70-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM70-NEXT: or.b32 %r86, %r82, 4194304;
+; SM70-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM70-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB236_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM70-LABEL: fsub_acq_rel_v2bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: $L__BB237_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: sub.rn.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: sub.rn.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM70-NEXT: mov.b32 %r24, %r1;
+; SM70-NEXT: @%p3 bra $L__BB237_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM70-LABEL: fsub_acq_rel_v4bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<47>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-NEXT: $L__BB238_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: sub.rn.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: sub.rn.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: sub.rn.f32 %r28, %r27, %r25;
+; SM70-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM70-NEXT: add.s32 %r30, %r29, %r28;
+; SM70-NEXT: add.s32 %r31, %r30, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM70-NEXT: or.b32 %r32, %r28, 4194304;
+; SM70-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: sub.rn.f32 %r38, %r37, %r35;
+; SM70-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM70-NEXT: add.s32 %r40, %r39, %r38;
+; SM70-NEXT: add.s32 %r41, %r40, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM70-NEXT: or.b32 %r42, %r38, 4194304;
+; SM70-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM70-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB238_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM70-LABEL: fsub_acq_rel_v8bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<93>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: $L__BB239_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: sub.rn.f32 %r9, %r8, %r6;
+; SM70-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM70-NEXT: add.s32 %r11, %r10, %r9;
+; SM70-NEXT: add.s32 %r12, %r11, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM70-NEXT: or.b32 %r13, %r9, 4194304;
+; SM70-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM70-NEXT: shl.b32 %r18, %r17, 16;
+; SM70-NEXT: sub.rn.f32 %r19, %r18, %r16;
+; SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM70-NEXT: add.s32 %r21, %r20, %r19;
+; SM70-NEXT: add.s32 %r22, %r21, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM70-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM70-NEXT: shl.b32 %r29, %r28, 16;
+; SM70-NEXT: sub.rn.f32 %r30, %r29, %r27;
+; SM70-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM70-NEXT: add.s32 %r32, %r31, %r30;
+; SM70-NEXT: add.s32 %r33, %r32, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM70-NEXT: or.b32 %r34, %r30, 4194304;
+; SM70-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM70-NEXT: shl.b32 %r39, %r38, 16;
+; SM70-NEXT: sub.rn.f32 %r40, %r39, %r37;
+; SM70-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM70-NEXT: add.s32 %r42, %r41, %r40;
+; SM70-NEXT: add.s32 %r43, %r42, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM70-NEXT: or.b32 %r44, %r40, 4194304;
+; SM70-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM70-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB239_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM70-NEXT: $L__BB239_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM70-NEXT: shl.b32 %r48, %r47, 16;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM70-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM70-NEXT: shl.b32 %r50, %r49, 16;
+; SM70-NEXT: sub.rn.f32 %r51, %r50, %r48;
+; SM70-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM70-NEXT: add.s32 %r53, %r52, %r51;
+; SM70-NEXT: add.s32 %r54, %r53, 32767;
+; SM70-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM70-NEXT: or.b32 %r55, %r51, 4194304;
+; SM70-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM70-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM70-NEXT: shl.b32 %r58, %r57, 16;
+; SM70-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM70-NEXT: shl.b32 %r60, %r59, 16;
+; SM70-NEXT: sub.rn.f32 %r61, %r60, %r58;
+; SM70-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM70-NEXT: add.s32 %r63, %r62, %r61;
+; SM70-NEXT: add.s32 %r64, %r63, 32767;
+; SM70-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM70-NEXT: or.b32 %r65, %r61, 4194304;
+; SM70-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM70-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM70-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM70-NEXT: shl.b32 %r69, %r68, 16;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM70-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM70-NEXT: shl.b32 %r71, %r70, 16;
+; SM70-NEXT: sub.rn.f32 %r72, %r71, %r69;
+; SM70-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM70-NEXT: add.s32 %r74, %r73, %r72;
+; SM70-NEXT: add.s32 %r75, %r74, 32767;
+; SM70-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM70-NEXT: or.b32 %r76, %r72, 4194304;
+; SM70-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM70-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM70-NEXT: shl.b32 %r79, %r78, 16;
+; SM70-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM70-NEXT: shl.b32 %r81, %r80, 16;
+; SM70-NEXT: sub.rn.f32 %r82, %r81, %r79;
+; SM70-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM70-NEXT: add.s32 %r84, %r83, %r82;
+; SM70-NEXT: add.s32 %r85, %r84, 32767;
+; SM70-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM70-NEXT: or.b32 %r86, %r82, 4194304;
+; SM70-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM70-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB239_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM70-LABEL: fmin_acq_rel_v2bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: $L__BB240_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: min.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: min.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM70-NEXT: mov.b32 %r24, %r1;
+; SM70-NEXT: @%p3 bra $L__BB240_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM70-LABEL: fmin_acq_rel_v4bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<47>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-NEXT: $L__BB241_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: min.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: min.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: min.f32 %r28, %r27, %r25;
+; SM70-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM70-NEXT: add.s32 %r30, %r29, %r28;
+; SM70-NEXT: add.s32 %r31, %r30, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM70-NEXT: or.b32 %r32, %r28, 4194304;
+; SM70-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: min.f32 %r38, %r37, %r35;
+; SM70-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM70-NEXT: add.s32 %r40, %r39, %r38;
+; SM70-NEXT: add.s32 %r41, %r40, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM70-NEXT: or.b32 %r42, %r38, 4194304;
+; SM70-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM70-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB241_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM70-LABEL: fmin_acq_rel_v8bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<93>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: $L__BB242_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: min.f32 %r9, %r8, %r6;
+; SM70-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM70-NEXT: add.s32 %r11, %r10, %r9;
+; SM70-NEXT: add.s32 %r12, %r11, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM70-NEXT: or.b32 %r13, %r9, 4194304;
+; SM70-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM70-NEXT: shl.b32 %r18, %r17, 16;
+; SM70-NEXT: min.f32 %r19, %r18, %r16;
+; SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM70-NEXT: add.s32 %r21, %r20, %r19;
+; SM70-NEXT: add.s32 %r22, %r21, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM70-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM70-NEXT: shl.b32 %r29, %r28, 16;
+; SM70-NEXT: min.f32 %r30, %r29, %r27;
+; SM70-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM70-NEXT: add.s32 %r32, %r31, %r30;
+; SM70-NEXT: add.s32 %r33, %r32, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM70-NEXT: or.b32 %r34, %r30, 4194304;
+; SM70-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM70-NEXT: shl.b32 %r39, %r38, 16;
+; SM70-NEXT: min.f32 %r40, %r39, %r37;
+; SM70-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM70-NEXT: add.s32 %r42, %r41, %r40;
+; SM70-NEXT: add.s32 %r43, %r42, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM70-NEXT: or.b32 %r44, %r40, 4194304;
+; SM70-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM70-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB242_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM70-NEXT: $L__BB242_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM70-NEXT: shl.b32 %r48, %r47, 16;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM70-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM70-NEXT: shl.b32 %r50, %r49, 16;
+; SM70-NEXT: min.f32 %r51, %r50, %r48;
+; SM70-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM70-NEXT: add.s32 %r53, %r52, %r51;
+; SM70-NEXT: add.s32 %r54, %r53, 32767;
+; SM70-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM70-NEXT: or.b32 %r55, %r51, 4194304;
+; SM70-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM70-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM70-NEXT: shl.b32 %r58, %r57, 16;
+; SM70-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM70-NEXT: shl.b32 %r60, %r59, 16;
+; SM70-NEXT: min.f32 %r61, %r60, %r58;
+; SM70-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM70-NEXT: add.s32 %r63, %r62, %r61;
+; SM70-NEXT: add.s32 %r64, %r63, 32767;
+; SM70-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM70-NEXT: or.b32 %r65, %r61, 4194304;
+; SM70-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM70-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM70-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM70-NEXT: shl.b32 %r69, %r68, 16;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM70-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM70-NEXT: shl.b32 %r71, %r70, 16;
+; SM70-NEXT: min.f32 %r72, %r71, %r69;
+; SM70-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM70-NEXT: add.s32 %r74, %r73, %r72;
+; SM70-NEXT: add.s32 %r75, %r74, 32767;
+; SM70-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM70-NEXT: or.b32 %r76, %r72, 4194304;
+; SM70-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM70-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM70-NEXT: shl.b32 %r79, %r78, 16;
+; SM70-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM70-NEXT: shl.b32 %r81, %r80, 16;
+; SM70-NEXT: min.f32 %r82, %r81, %r79;
+; SM70-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM70-NEXT: add.s32 %r84, %r83, %r82;
+; SM70-NEXT: add.s32 %r85, %r84, 32767;
+; SM70-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM70-NEXT: or.b32 %r86, %r82, 4194304;
+; SM70-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM70-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB242_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM70-LABEL: fmax_acq_rel_v2bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: $L__BB243_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: max.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: max.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM70-NEXT: mov.b32 %r24, %r1;
+; SM70-NEXT: @%p3 bra $L__BB243_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM70-LABEL: fmax_acq_rel_v4bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<47>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-NEXT: $L__BB244_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: max.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: max.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: max.f32 %r28, %r27, %r25;
+; SM70-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM70-NEXT: add.s32 %r30, %r29, %r28;
+; SM70-NEXT: add.s32 %r31, %r30, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM70-NEXT: or.b32 %r32, %r28, 4194304;
+; SM70-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: max.f32 %r38, %r37, %r35;
+; SM70-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM70-NEXT: add.s32 %r40, %r39, %r38;
+; SM70-NEXT: add.s32 %r41, %r40, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM70-NEXT: or.b32 %r42, %r38, 4194304;
+; SM70-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM70-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB244_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM70-LABEL: fmax_acq_rel_v8bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<93>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: $L__BB245_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: max.f32 %r9, %r8, %r6;
+; SM70-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM70-NEXT: add.s32 %r11, %r10, %r9;
+; SM70-NEXT: add.s32 %r12, %r11, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM70-NEXT: or.b32 %r13, %r9, 4194304;
+; SM70-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM70-NEXT: shl.b32 %r18, %r17, 16;
+; SM70-NEXT: max.f32 %r19, %r18, %r16;
+; SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM70-NEXT: add.s32 %r21, %r20, %r19;
+; SM70-NEXT: add.s32 %r22, %r21, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM70-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM70-NEXT: shl.b32 %r29, %r28, 16;
+; SM70-NEXT: max.f32 %r30, %r29, %r27;
+; SM70-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM70-NEXT: add.s32 %r32, %r31, %r30;
+; SM70-NEXT: add.s32 %r33, %r32, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM70-NEXT: or.b32 %r34, %r30, 4194304;
+; SM70-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM70-NEXT: shl.b32 %r39, %r38, 16;
+; SM70-NEXT: max.f32 %r40, %r39, %r37;
+; SM70-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM70-NEXT: add.s32 %r42, %r41, %r40;
+; SM70-NEXT: add.s32 %r43, %r42, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM70-NEXT: or.b32 %r44, %r40, 4194304;
+; SM70-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM70-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB245_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM70-NEXT: $L__BB245_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM70-NEXT: shl.b32 %r48, %r47, 16;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM70-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM70-NEXT: shl.b32 %r50, %r49, 16;
+; SM70-NEXT: max.f32 %r51, %r50, %r48;
+; SM70-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM70-NEXT: add.s32 %r53, %r52, %r51;
+; SM70-NEXT: add.s32 %r54, %r53, 32767;
+; SM70-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM70-NEXT: or.b32 %r55, %r51, 4194304;
+; SM70-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM70-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM70-NEXT: shl.b32 %r58, %r57, 16;
+; SM70-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM70-NEXT: shl.b32 %r60, %r59, 16;
+; SM70-NEXT: max.f32 %r61, %r60, %r58;
+; SM70-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM70-NEXT: add.s32 %r63, %r62, %r61;
+; SM70-NEXT: add.s32 %r64, %r63, 32767;
+; SM70-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM70-NEXT: or.b32 %r65, %r61, 4194304;
+; SM70-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM70-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM70-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM70-NEXT: shl.b32 %r69, %r68, 16;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM70-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM70-NEXT: shl.b32 %r71, %r70, 16;
+; SM70-NEXT: max.f32 %r72, %r71, %r69;
+; SM70-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM70-NEXT: add.s32 %r74, %r73, %r72;
+; SM70-NEXT: add.s32 %r75, %r74, 32767;
+; SM70-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM70-NEXT: or.b32 %r76, %r72, 4194304;
+; SM70-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM70-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM70-NEXT: shl.b32 %r79, %r78, 16;
+; SM70-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM70-NEXT: shl.b32 %r81, %r80, 16;
+; SM70-NEXT: max.f32 %r82, %r81, %r79;
+; SM70-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM70-NEXT: add.s32 %r84, %r83, %r82;
+; SM70-NEXT: add.s32 %r85, %r84, 32767;
+; SM70-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM70-NEXT: or.b32 %r86, %r82, 4194304;
+; SM70-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM70-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB245_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM70-LABEL: fminimum_acq_rel_v2bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<12>;
+; SM70-NEXT: .reg .b16 %rs<15>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM70-NEXT: $L__BB246_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r16;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: setp.lt.f32 %p1, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NEXT: setp.lt.f32 %p6, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM70-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
+; SM70-NEXT: setp.ne.b32 %p11, %r1, %r16;
+; SM70-NEXT: mov.b32 %r16, %r1;
+; SM70-NEXT: @%p11 bra $L__BB246_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM70-LABEL: fminimum_acq_rel_v4bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<22>;
+; SM70-NEXT: .reg .b16 %rs<29>;
+; SM70-NEXT: .reg .b32 %r<31>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM70-NEXT: shl.b32 %r17, %r16, 16;
+; SM70-NEXT: setp.eq.b16 %p14, %rs16, -32768;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs15;
+; SM70-NEXT: shl.b32 %r23, %r22, 16;
+; SM70-NEXT: setp.eq.b16 %p19, %rs15, -32768;
+; SM70-NEXT: $L__BB247_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r29;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: setp.lt.f32 %p1, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NEXT: setp.lt.f32 %p6, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM70-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r15;
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r30;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs18;
+; SM70-NEXT: shl.b32 %r19, %r18, 16;
+; SM70-NEXT: setp.lt.f32 %p11, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM70-NEXT: setp.nan.f32 %p12, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs18, -32768;
+; SM70-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM70-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM70-NEXT: shl.b32 %r21, %r20, 16;
+; SM70-NEXT: setp.eq.f32 %p15, %r21, 0f00000000;
+; SM70-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs17;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: setp.lt.f32 %p16, %r25, %r23;
+; SM70-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM70-NEXT: setp.nan.f32 %p17, %r25, %r23;
+; SM70-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs17, -32768;
+; SM70-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM70-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs25;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: setp.eq.f32 %p20, %r27, 0f00000000;
+; SM70-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM70-NEXT: mov.b32 %r28, {%rs28, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r28;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r29;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r30;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB247_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM70-LABEL: fminimum_acq_rel_v8bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<43>;
+; SM70-NEXT: .reg .b16 %rs<57>;
+; SM70-NEXT: .reg .b32 %r<61>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r57, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs16;
+; SM70-NEXT: shl.b32 %r19, %r18, 16;
+; SM70-NEXT: setp.eq.b16 %p14, %rs16, -32768;
+; SM70-NEXT: setp.eq.b16 %p19, %rs15, -32768;
+; SM70-NEXT: $L__BB248_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r57;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: setp.lt.f32 %p1, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r10, 0f00000000;
+; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs1;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs3;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: setp.lt.f32 %p6, %r14, %r12;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f32 %p7, %r14, %r12;
+; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs11;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: setp.eq.f32 %p10, %r16, 0f00000000;
+; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM70-NEXT: mov.b32 %r17, {%rs14, %rs9};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r58;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM70-NEXT: shl.b32 %r21, %r20, 16;
+; SM70-NEXT: setp.lt.f32 %p11, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM70-NEXT: setp.nan.f32 %p12, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs18, -32768;
+; SM70-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM70-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs20;
+; SM70-NEXT: shl.b32 %r23, %r22, 16;
+; SM70-NEXT: setp.eq.f32 %p15, %r23, 0f00000000;
+; SM70-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs15;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs17;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: setp.lt.f32 %p16, %r27, %r25;
+; SM70-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM70-NEXT: setp.nan.f32 %p17, %r27, %r25;
+; SM70-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs17, -32768;
+; SM70-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM70-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM70-NEXT: cvt.u32.u16 %r28, %rs25;
+; SM70-NEXT: shl.b32 %r29, %r28, 16;
+; SM70-NEXT: setp.eq.f32 %p20, %r29, 0f00000000;
+; SM70-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM70-NEXT: mov.b32 %r30, {%rs28, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r30;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r57;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r58;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r57, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB248_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r59, %rd12;
+; SM70-NEXT: $L__BB248_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs29, %rs30}, %r3;
+; SM70-NEXT: cvt.u32.u16 %r31, %rs30;
+; SM70-NEXT: shl.b32 %r32, %r31, 16;
+; SM70-NEXT: mov.b32 {%rs31, %rs32}, %r59;
+; SM70-NEXT: cvt.u32.u16 %r33, %rs32;
+; SM70-NEXT: shl.b32 %r34, %r33, 16;
+; SM70-NEXT: setp.lt.f32 %p22, %r34, %r32;
+; SM70-NEXT: selp.b16 %rs33, %rs32, %rs30, %p22;
+; SM70-NEXT: setp.nan.f32 %p23, %r34, %r32;
+; SM70-NEXT: selp.b16 %rs34, 0x7FC0, %rs33, %p23;
+; SM70-NEXT: setp.eq.b16 %p24, %rs32, -32768;
+; SM70-NEXT: selp.b16 %rs35, %rs32, %rs34, %p24;
+; SM70-NEXT: setp.eq.b16 %p25, %rs30, -32768;
+; SM70-NEXT: selp.b16 %rs36, %rs30, %rs35, %p25;
+; SM70-NEXT: cvt.u32.u16 %r35, %rs34;
+; SM70-NEXT: shl.b32 %r36, %r35, 16;
+; SM70-NEXT: setp.eq.f32 %p26, %r36, 0f00000000;
+; SM70-NEXT: selp.b16 %rs37, %rs36, %rs34, %p26;
+; SM70-NEXT: cvt.u32.u16 %r37, %rs29;
+; SM70-NEXT: shl.b32 %r38, %r37, 16;
+; SM70-NEXT: cvt.u32.u16 %r39, %rs31;
+; SM70-NEXT: shl.b32 %r40, %r39, 16;
+; SM70-NEXT: setp.lt.f32 %p27, %r40, %r38;
+; SM70-NEXT: selp.b16 %rs38, %rs31, %rs29, %p27;
+; SM70-NEXT: setp.nan.f32 %p28, %r40, %r38;
+; SM70-NEXT: selp.b16 %rs39, 0x7FC0, %rs38, %p28;
+; SM70-NEXT: setp.eq.b16 %p29, %rs31, -32768;
+; SM70-NEXT: selp.b16 %rs40, %rs31, %rs39, %p29;
+; SM70-NEXT: setp.eq.b16 %p30, %rs29, -32768;
+; SM70-NEXT: selp.b16 %rs41, %rs29, %rs40, %p30;
+; SM70-NEXT: cvt.u32.u16 %r41, %rs39;
+; SM70-NEXT: shl.b32 %r42, %r41, 16;
+; SM70-NEXT: setp.eq.f32 %p31, %r42, 0f00000000;
+; SM70-NEXT: selp.b16 %rs42, %rs41, %rs39, %p31;
+; SM70-NEXT: mov.b32 %r43, {%rs42, %rs37};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r43;
+; SM70-NEXT: mov.b32 {%rs43, %rs44}, %r4;
+; SM70-NEXT: cvt.u32.u16 %r44, %rs44;
+; SM70-NEXT: shl.b32 %r45, %r44, 16;
+; SM70-NEXT: mov.b32 {%rs45, %rs46}, %r60;
+; SM70-NEXT: cvt.u32.u16 %r46, %rs46;
+; SM70-NEXT: shl.b32 %r47, %r46, 16;
+; SM70-NEXT: setp.lt.f32 %p32, %r47, %r45;
+; SM70-NEXT: selp.b16 %rs47, %rs46, %rs44, %p32;
+; SM70-NEXT: setp.nan.f32 %p33, %r47, %r45;
+; SM70-NEXT: selp.b16 %rs48, 0x7FC0, %rs47, %p33;
+; SM70-NEXT: setp.eq.b16 %p34, %rs46, -32768;
+; SM70-NEXT: selp.b16 %rs49, %rs46, %rs48, %p34;
+; SM70-NEXT: setp.eq.b16 %p35, %rs44, -32768;
+; SM70-NEXT: selp.b16 %rs50, %rs44, %rs49, %p35;
+; SM70-NEXT: cvt.u32.u16 %r48, %rs48;
+; SM70-NEXT: shl.b32 %r49, %r48, 16;
+; SM70-NEXT: setp.eq.f32 %p36, %r49, 0f00000000;
+; SM70-NEXT: selp.b16 %rs51, %rs50, %rs48, %p36;
+; SM70-NEXT: cvt.u32.u16 %r50, %rs43;
+; SM70-NEXT: shl.b32 %r51, %r50, 16;
+; SM70-NEXT: cvt.u32.u16 %r52, %rs45;
+; SM70-NEXT: shl.b32 %r53, %r52, 16;
+; SM70-NEXT: setp.lt.f32 %p37, %r53, %r51;
+; SM70-NEXT: selp.b16 %rs52, %rs45, %rs43, %p37;
+; SM70-NEXT: setp.nan.f32 %p38, %r53, %r51;
+; SM70-NEXT: selp.b16 %rs53, 0x7FC0, %rs52, %p38;
+; SM70-NEXT: setp.eq.b16 %p39, %rs45, -32768;
+; SM70-NEXT: selp.b16 %rs54, %rs45, %rs53, %p39;
+; SM70-NEXT: setp.eq.b16 %p40, %rs43, -32768;
+; SM70-NEXT: selp.b16 %rs55, %rs43, %rs54, %p40;
+; SM70-NEXT: cvt.u32.u16 %r54, %rs53;
+; SM70-NEXT: shl.b32 %r55, %r54, 16;
+; SM70-NEXT: setp.eq.f32 %p41, %r55, 0f00000000;
+; SM70-NEXT: selp.b16 %rs56, %rs55, %rs53, %p41;
+; SM70-NEXT: mov.b32 %r56, {%rs56, %rs51};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r56;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r59;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r60;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r59, %rd21;
+; SM70-NEXT: @%p42 bra $L__BB248_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<12>;
+; SM70-NEXT: .reg .b16 %rs<15>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM70-NEXT: $L__BB249_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r16;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: setp.gt.f32 %p1, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NEXT: setp.gt.f32 %p6, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM70-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
+; SM70-NEXT: setp.ne.b32 %p11, %r1, %r16;
+; SM70-NEXT: mov.b32 %r16, %r1;
+; SM70-NEXT: @%p11 bra $L__BB249_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v4bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<22>;
+; SM70-NEXT: .reg .b16 %rs<29>;
+; SM70-NEXT: .reg .b32 %r<31>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM70-NEXT: shl.b32 %r17, %r16, 16;
+; SM70-NEXT: setp.eq.b16 %p14, %rs16, 0;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs15;
+; SM70-NEXT: shl.b32 %r23, %r22, 16;
+; SM70-NEXT: setp.eq.b16 %p19, %rs15, 0;
+; SM70-NEXT: $L__BB250_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r29;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: setp.gt.f32 %p1, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NEXT: setp.gt.f32 %p6, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM70-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r15;
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r30;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs18;
+; SM70-NEXT: shl.b32 %r19, %r18, 16;
+; SM70-NEXT: setp.gt.f32 %p11, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM70-NEXT: setp.nan.f32 %p12, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs18, 0;
+; SM70-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM70-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM70-NEXT: shl.b32 %r21, %r20, 16;
+; SM70-NEXT: setp.eq.f32 %p15, %r21, 0f00000000;
+; SM70-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs17;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: setp.gt.f32 %p16, %r25, %r23;
+; SM70-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM70-NEXT: setp.nan.f32 %p17, %r25, %r23;
+; SM70-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs17, 0;
+; SM70-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM70-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs25;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: setp.eq.f32 %p20, %r27, 0f00000000;
+; SM70-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM70-NEXT: mov.b32 %r28, {%rs28, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r28;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r29;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r30;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB250_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v8bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<43>;
+; SM70-NEXT: .reg .b16 %rs<57>;
+; SM70-NEXT: .reg .b32 %r<61>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r57, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs16;
+; SM70-NEXT: shl.b32 %r19, %r18, 16;
+; SM70-NEXT: setp.eq.b16 %p14, %rs16, 0;
+; SM70-NEXT: setp.eq.b16 %p19, %rs15, 0;
+; SM70-NEXT: $L__BB251_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r57;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: setp.gt.f32 %p1, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r10, 0f00000000;
+; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs1;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs3;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: setp.gt.f32 %p6, %r14, %r12;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f32 %p7, %r14, %r12;
+; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs11;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: setp.eq.f32 %p10, %r16, 0f00000000;
+; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM70-NEXT: mov.b32 %r17, {%rs14, %rs9};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r58;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM70-NEXT: shl.b32 %r21, %r20, 16;
+; SM70-NEXT: setp.gt.f32 %p11, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM70-NEXT: setp.nan.f32 %p12, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs18, 0;
+; SM70-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM70-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs20;
+; SM70-NEXT: shl.b32 %r23, %r22, 16;
+; SM70-NEXT: setp.eq.f32 %p15, %r23, 0f00000000;
+; SM70-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs15;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs17;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: setp.gt.f32 %p16, %r27, %r25;
+; SM70-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM70-NEXT: setp.nan.f32 %p17, %r27, %r25;
+; SM70-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs17, 0;
+; SM70-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM70-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM70-NEXT: cvt.u32.u16 %r28, %rs25;
+; SM70-NEXT: shl.b32 %r29, %r28, 16;
+; SM70-NEXT: setp.eq.f32 %p20, %r29, 0f00000000;
+; SM70-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM70-NEXT: mov.b32 %r30, {%rs28, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r30;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r57;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r58;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r57, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB251_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r59, %rd12;
+; SM70-NEXT: $L__BB251_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs29, %rs30}, %r3;
+; SM70-NEXT: cvt.u32.u16 %r31, %rs30;
+; SM70-NEXT: shl.b32 %r32, %r31, 16;
+; SM70-NEXT: mov.b32 {%rs31, %rs32}, %r59;
+; SM70-NEXT: cvt.u32.u16 %r33, %rs32;
+; SM70-NEXT: shl.b32 %r34, %r33, 16;
+; SM70-NEXT: setp.gt.f32 %p22, %r34, %r32;
+; SM70-NEXT: selp.b16 %rs33, %rs32, %rs30, %p22;
+; SM70-NEXT: setp.nan.f32 %p23, %r34, %r32;
+; SM70-NEXT: selp.b16 %rs34, 0x7FC0, %rs33, %p23;
+; SM70-NEXT: setp.eq.b16 %p24, %rs32, 0;
+; SM70-NEXT: selp.b16 %rs35, %rs32, %rs34, %p24;
+; SM70-NEXT: setp.eq.b16 %p25, %rs30, 0;
+; SM70-NEXT: selp.b16 %rs36, %rs30, %rs35, %p25;
+; SM70-NEXT: cvt.u32.u16 %r35, %rs34;
+; SM70-NEXT: shl.b32 %r36, %r35, 16;
+; SM70-NEXT: setp.eq.f32 %p26, %r36, 0f00000000;
+; SM70-NEXT: selp.b16 %rs37, %rs36, %rs34, %p26;
+; SM70-NEXT: cvt.u32.u16 %r37, %rs29;
+; SM70-NEXT: shl.b32 %r38, %r37, 16;
+; SM70-NEXT: cvt.u32.u16 %r39, %rs31;
+; SM70-NEXT: shl.b32 %r40, %r39, 16;
+; SM70-NEXT: setp.gt.f32 %p27, %r40, %r38;
+; SM70-NEXT: selp.b16 %rs38, %rs31, %rs29, %p27;
+; SM70-NEXT: setp.nan.f32 %p28, %r40, %r38;
+; SM70-NEXT: selp.b16 %rs39, 0x7FC0, %rs38, %p28;
+; SM70-NEXT: setp.eq.b16 %p29, %rs31, 0;
+; SM70-NEXT: selp.b16 %rs40, %rs31, %rs39, %p29;
+; SM70-NEXT: setp.eq.b16 %p30, %rs29, 0;
+; SM70-NEXT: selp.b16 %rs41, %rs29, %rs40, %p30;
+; SM70-NEXT: cvt.u32.u16 %r41, %rs39;
+; SM70-NEXT: shl.b32 %r42, %r41, 16;
+; SM70-NEXT: setp.eq.f32 %p31, %r42, 0f00000000;
+; SM70-NEXT: selp.b16 %rs42, %rs41, %rs39, %p31;
+; SM70-NEXT: mov.b32 %r43, {%rs42, %rs37};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r43;
+; SM70-NEXT: mov.b32 {%rs43, %rs44}, %r4;
+; SM70-NEXT: cvt.u32.u16 %r44, %rs44;
+; SM70-NEXT: shl.b32 %r45, %r44, 16;
+; SM70-NEXT: mov.b32 {%rs45, %rs46}, %r60;
+; SM70-NEXT: cvt.u32.u16 %r46, %rs46;
+; SM70-NEXT: shl.b32 %r47, %r46, 16;
+; SM70-NEXT: setp.gt.f32 %p32, %r47, %r45;
+; SM70-NEXT: selp.b16 %rs47, %rs46, %rs44, %p32;
+; SM70-NEXT: setp.nan.f32 %p33, %r47, %r45;
+; SM70-NEXT: selp.b16 %rs48, 0x7FC0, %rs47, %p33;
+; SM70-NEXT: setp.eq.b16 %p34, %rs46, 0;
+; SM70-NEXT: selp.b16 %rs49, %rs46, %rs48, %p34;
+; SM70-NEXT: setp.eq.b16 %p35, %rs44, 0;
+; SM70-NEXT: selp.b16 %rs50, %rs44, %rs49, %p35;
+; SM70-NEXT: cvt.u32.u16 %r48, %rs48;
+; SM70-NEXT: shl.b32 %r49, %r48, 16;
+; SM70-NEXT: setp.eq.f32 %p36, %r49, 0f00000000;
+; SM70-NEXT: selp.b16 %rs51, %rs50, %rs48, %p36;
+; SM70-NEXT: cvt.u32.u16 %r50, %rs43;
+; SM70-NEXT: shl.b32 %r51, %r50, 16;
+; SM70-NEXT: cvt.u32.u16 %r52, %rs45;
+; SM70-NEXT: shl.b32 %r53, %r52, 16;
+; SM70-NEXT: setp.gt.f32 %p37, %r53, %r51;
+; SM70-NEXT: selp.b16 %rs52, %rs45, %rs43, %p37;
+; SM70-NEXT: setp.nan.f32 %p38, %r53, %r51;
+; SM70-NEXT: selp.b16 %rs53, 0x7FC0, %rs52, %p38;
+; SM70-NEXT: setp.eq.b16 %p39, %rs45, 0;
+; SM70-NEXT: selp.b16 %rs54, %rs45, %rs53, %p39;
+; SM70-NEXT: setp.eq.b16 %p40, %rs43, 0;
+; SM70-NEXT: selp.b16 %rs55, %rs43, %rs54, %p40;
+; SM70-NEXT: cvt.u32.u16 %r54, %rs53;
+; SM70-NEXT: shl.b32 %r55, %r54, 16;
+; SM70-NEXT: setp.eq.f32 %p41, %r55, 0f00000000;
+; SM70-NEXT: selp.b16 %rs56, %rs55, %rs53, %p41;
+; SM70-NEXT: mov.b32 %r56, {%rs56, %rs51};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r56;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r59;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r60;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r59, %rd21;
+; SM70-NEXT: @%p42 bra $L__BB251_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_monotonic_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_monotonic_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [add_monotonic_v2i8_global_cta_param_0];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB252_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB252_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_acquire_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB253_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB253_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_release_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB254_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB254_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_seq_cst_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB255_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB255_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: add_monotonic_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_monotonic_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: $L__BB256_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB256_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: add_acquire_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: $L__BB257_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB257_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: add_release_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: $L__BB258_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB258_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: add_seq_cst_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: $L__BB259_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB259_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: add_monotonic_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB260_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM70-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM70-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM70-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB260_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: add_acquire_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB261_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM70-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM70-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM70-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB261_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: add_release_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB262_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM70-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM70-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM70-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB262_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: add_seq_cst_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB263_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM70-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM70-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM70-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB263_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
+}
+
+define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: add_monotonic_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v2i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: add_acquire_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: add_release_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: add_seq_cst_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_monotonic_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_acquire_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v4i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_release_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_seq_cst_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: add_monotonic_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: add_acquire_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: add_release_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_release_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: add_seq_cst_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_seq_cst_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
+ ret <8 x i32> %retval
+}
+
+define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: nand_monotonic_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_monotonic_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [nand_monotonic_v2i8_global_cta_param_0];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: $L__BB276_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r18, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, %r4;
+; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM70-NEXT: shl.b32 %r14, %r13, %r1;
+; SM70-NEXT: and.b32 %r15, %r18, %r2;
+; SM70-NEXT: or.b32 %r16, %r15, %r14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: @%p1 bra $L__BB276_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r17, %r3, %r1;
+; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: nand_acquire_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acquire_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [nand_acquire_v2i8_global_cta_param_0];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: $L__BB277_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r18, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, %r4;
+; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM70-NEXT: shl.b32 %r14, %r13, %r1;
+; SM70-NEXT: and.b32 %r15, %r18, %r2;
+; SM70-NEXT: or.b32 %r16, %r15, %r14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: @%p1 bra $L__BB277_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r17, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: nand_release_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: $L__BB278_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r18, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, %r4;
+; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM70-NEXT: shl.b32 %r14, %r13, %r1;
+; SM70-NEXT: and.b32 %r15, %r18, %r2;
+; SM70-NEXT: or.b32 %r16, %r15, %r14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: @%p1 bra $L__BB278_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r17, %r3, %r1;
+; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: nand_seq_cst_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: $L__BB279_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r18, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, %r4;
+; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM70-NEXT: shl.b32 %r14, %r13, %r1;
+; SM70-NEXT: and.b32 %r15, %r18, %r2;
+; SM70-NEXT: or.b32 %r16, %r15, %r14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: @%p1 bra $L__BB279_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r17, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_monotonic_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB280_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB280_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_acquire_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB281_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB281_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_release_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB282_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB282_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_seq_cst_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_seq_cst_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB283_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB283_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_monotonic_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB284_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB284_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_acquire_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB285_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB285_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_release_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB286_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB286_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_seq_cst_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB287_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB287_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
+}
+
+define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_monotonic_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB288_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB288_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_acquire_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB289_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB289_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_release_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB290_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB290_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_seq_cst_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB291_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB291_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_monotonic_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB292_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r10, %r2;
+; SM70-NEXT: and.b32 %r6, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB292_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM70-NEXT: $L__BB292_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r7, %r12, %r4;
+; SM70-NEXT: and.b32 %r8, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB292_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_acquire_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB293_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r10, %r2;
+; SM70-NEXT: and.b32 %r6, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB293_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM70-NEXT: $L__BB293_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r7, %r12, %r4;
+; SM70-NEXT: and.b32 %r8, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB293_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_release_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB294_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r10, %r2;
+; SM70-NEXT: and.b32 %r6, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB294_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM70-NEXT: $L__BB294_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r7, %r12, %r4;
+; SM70-NEXT: and.b32 %r8, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB294_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_seq_cst_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB295_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r10, %r2;
+; SM70-NEXT: and.b32 %r6, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB295_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM70-NEXT: $L__BB295_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r7, %r12, %r4;
+; SM70-NEXT: and.b32 %r8, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB295_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: nand_monotonic_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<46>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_monotonic_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB296_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r9, %r18, %r6;
+; SM70-NEXT: and.b32 %r10, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB296_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM70-NEXT: $L__BB296_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r11, %r20, %r8;
+; SM70-NEXT: and.b32 %r12, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB296_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM70-NEXT: $L__BB296_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM70-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM70-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM70-NEXT: and.b32 %r13, %r22, %r2;
+; SM70-NEXT: and.b32 %r14, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM70-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM70-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM70-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM70-NEXT: not.b64 %rd33, %rd32;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM70-NEXT: @%p3 bra $L__BB296_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM70-NEXT: $L__BB296_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM70-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM70-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM70-NEXT: and.b32 %r15, %r24, %r4;
+; SM70-NEXT: and.b32 %r16, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM70-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM70-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM70-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM70-NEXT: not.b64 %rd44, %rd43;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM70-NEXT: @%p4 bra $L__BB296_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: nand_acquire_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<46>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acquire_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r9, %r18, %r6;
+; SM70-NEXT: and.b32 %r10, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB297_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM70-NEXT: $L__BB297_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r11, %r20, %r8;
+; SM70-NEXT: and.b32 %r12, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB297_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM70-NEXT: $L__BB297_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM70-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM70-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM70-NEXT: and.b32 %r13, %r22, %r2;
+; SM70-NEXT: and.b32 %r14, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM70-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM70-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM70-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM70-NEXT: not.b64 %rd33, %rd32;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM70-NEXT: @%p3 bra $L__BB297_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM70-NEXT: $L__BB297_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM70-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM70-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM70-NEXT: and.b32 %r15, %r24, %r4;
+; SM70-NEXT: and.b32 %r16, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM70-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM70-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM70-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM70-NEXT: not.b64 %rd44, %rd43;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM70-NEXT: @%p4 bra $L__BB297_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: nand_release_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<46>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_release_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB298_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r9, %r18, %r6;
+; SM70-NEXT: and.b32 %r10, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB298_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM70-NEXT: $L__BB298_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r11, %r20, %r8;
+; SM70-NEXT: and.b32 %r12, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB298_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM70-NEXT: $L__BB298_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM70-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM70-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM70-NEXT: and.b32 %r13, %r22, %r2;
+; SM70-NEXT: and.b32 %r14, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM70-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM70-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM70-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM70-NEXT: not.b64 %rd33, %rd32;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM70-NEXT: @%p3 bra $L__BB298_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM70-NEXT: $L__BB298_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM70-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM70-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM70-NEXT: and.b32 %r15, %r24, %r4;
+; SM70-NEXT: and.b32 %r16, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM70-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM70-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM70-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM70-NEXT: not.b64 %rd44, %rd43;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM70-NEXT: @%p4 bra $L__BB298_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: nand_seq_cst_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<46>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_seq_cst_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB299_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r9, %r18, %r6;
+; SM70-NEXT: and.b32 %r10, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB299_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM70-NEXT: $L__BB299_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r11, %r20, %r8;
+; SM70-NEXT: and.b32 %r12, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB299_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM70-NEXT: $L__BB299_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM70-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM70-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM70-NEXT: and.b32 %r13, %r22, %r2;
+; SM70-NEXT: and.b32 %r14, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM70-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM70-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM70-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM70-NEXT: not.b64 %rd33, %rd32;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM70-NEXT: @%p3 bra $L__BB299_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM70-NEXT: $L__BB299_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM70-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM70-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM70-NEXT: and.b32 %r15, %r24, %r4;
+; SM70-NEXT: and.b32 %r16, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM70-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM70-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM70-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM70-NEXT: not.b64 %rd44, %rd43;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM70-NEXT: @%p4 bra $L__BB299_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
+ ret <8 x i32> %retval
+}
+
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
new file mode 100644
index 0000000000000..a868ebffa4f9c
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
@@ -0,0 +1,15129 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefix=SM90
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %}
+
+define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: xchg_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<14>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
+; SM90-NEXT: mov.b32 %r8, 65535;
+; SM90-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-NEXT: not.b32 %r2, %r9;
+; SM90-NEXT: shl.b32 %r3, %r5, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-NEXT: $L__BB0_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-NEXT: or.b32 %r11, %r10, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM90-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM90-NEXT: mov.b32 %r13, %r4;
+; SM90-NEXT: @%p1 bra $L__BB0_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r12, %r4, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r12;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @xchg_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: xchg_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @xchg_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: xchg_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @xchg_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: xchg_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @xchg_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: xchg_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @xchg_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: xchg_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 amt, dst;
+; SM90-NEXT: mov.b128 amt, {%rd2, %rd3};
+; SM90-NEXT: atom.acq_rel.cta.global.exch.b128 dst, [%rd1], amt;
+; SM90-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @xchg_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: xchg_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [xchg_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @xchg_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: xchg_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @xchg_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: xchg_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xchg_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @xchg_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: xchg_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @xchg_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: xchg_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @xchg_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: xchg_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xchg_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xchg_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: add_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<10>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM90-NEXT: $L__BB12_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: and.b16 %rs8, %rs5, 255;
+; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: shl.b32 %r13, %r12, %r1;
+; SM90-NEXT: and.b32 %r14, %r17, %r2;
+; SM90-NEXT: or.b32 %r15, %r14, %r13;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM90-NEXT: mov.b32 %r17, %r3;
+; SM90-NEXT: @%p1 bra $L__BB12_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r16, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [add_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: $L__BB13_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB13_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: add_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB14_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB14_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: add_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [add_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB15_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB15_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: add_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB16_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: add.s16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB16_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: add_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB17_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: add.s16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: add.s16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: add.s16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB17_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @add_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: add_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: add_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: add_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @add_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: add_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @add_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: add_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @add_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: add_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [add_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [add_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: sub_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<12>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [sub_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM90-NEXT: $L__BB24_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r15, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: sub.s16 %rs7, %rs4, %rs6;
+; SM90-NEXT: sub.s16 %rs8, %rs3, %rs5;
+; SM90-NEXT: and.b16 %rs9, %rs8, 255;
+; SM90-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM90-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p1 bra $L__BB24_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: sub_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [sub_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: $L__BB25_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: sub.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: sub.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB25_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: sub_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB26_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: sub.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: sub.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: sub.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: sub.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: sub.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: sub.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB26_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: sub_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<7>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [sub_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM90-NEXT: $L__BB27_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM90-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM90-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM90-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB27_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: sub_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM90-NEXT: $L__BB28_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM90-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM90-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM90-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM90-NEXT: sub.s16 %rs11, %rs10, %rs8;
+; SM90-NEXT: sub.s16 %rs12, %rs9, %rs7;
+; SM90-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB28_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: sub_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM90-NEXT: $L__BB29_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM90-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM90-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM90-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM90-NEXT: sub.s16 %rs11, %rs10, %rs8;
+; SM90-NEXT: sub.s16 %rs12, %rs9, %rs7;
+; SM90-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM90-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM90-NEXT: sub.s16 %rs17, %rs16, %rs14;
+; SM90-NEXT: sub.s16 %rs18, %rs15, %rs13;
+; SM90-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM90-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM90-NEXT: sub.s16 %rs23, %rs22, %rs20;
+; SM90-NEXT: sub.s16 %rs24, %rs21, %rs19;
+; SM90-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB29_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: sub_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: neg.s32 %r3, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1], %r3;
+; SM90-NEXT: neg.s32 %r5, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r5;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @sub_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: sub_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: neg.s32 %r5, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1], %r5;
+; SM90-NEXT: neg.s32 %r7, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+4], %r7;
+; SM90-NEXT: neg.s32 %r9, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+8], %r9;
+; SM90-NEXT: neg.s32 %r11, %r4;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r11;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: sub_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: neg.s32 %r9, %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1], %r9;
+; SM90-NEXT: neg.s32 %r11, %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+4], %r11;
+; SM90-NEXT: neg.s32 %r13, %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+8], %r13;
+; SM90-NEXT: neg.s32 %r15, %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+12], %r15;
+; SM90-NEXT: neg.s32 %r17, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r18, [%rd1+16], %r17;
+; SM90-NEXT: neg.s32 %r19, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r20, [%rd1+20], %r19;
+; SM90-NEXT: neg.s32 %r21, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r22, [%rd1+24], %r21;
+; SM90-NEXT: neg.s32 %r23, %r4;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r24, [%rd1+28], %r23;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: sub_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<8>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: neg.s64 %rd4, %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd5, [%rd1], %rd4;
+; SM90-NEXT: neg.s64 %rd6, %rd3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd6;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @sub_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: sub_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<14>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: neg.s64 %rd6, %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1], %rd6;
+; SM90-NEXT: neg.s64 %rd8, %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd9, [%rd1+8], %rd8;
+; SM90-NEXT: neg.s64 %rd10, %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1+16], %rd10;
+; SM90-NEXT: neg.s64 %rd12, %rd3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd12;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd11, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd7, %rd9};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: sub_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [sub_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: neg.s64 %rd10, %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1], %rd10;
+; SM90-NEXT: neg.s64 %rd12, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+8], %rd12;
+; SM90-NEXT: neg.s64 %rd14, %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd15, [%rd1+16], %rd14;
+; SM90-NEXT: neg.s64 %rd16, %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd17, [%rd1+24], %rd16;
+; SM90-NEXT: neg.s64 %rd18, %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd19, [%rd1+32], %rd18;
+; SM90-NEXT: neg.s64 %rd20, %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd21, [%rd1+40], %rd20;
+; SM90-NEXT: neg.s64 %rd22, %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd23, [%rd1+48], %rd22;
+; SM90-NEXT: neg.s64 %rd24, %rd3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd25, [%rd1+56], %rd24;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd23, %rd25};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd19, %rd21};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd15, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd11, %rd13};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @and_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: and_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<12>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM90-NEXT: and.b32 %r2, %r1, 3;
+; SM90-NEXT: shl.b32 %r3, %r2, 3;
+; SM90-NEXT: mov.b32 %r4, 65535;
+; SM90-NEXT: shl.b32 %r5, %r4, %r3;
+; SM90-NEXT: not.b32 %r6, %r5;
+; SM90-NEXT: ld.param.b16 %r7, [and_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r8, %r7, %r3;
+; SM90-NEXT: or.b32 %r9, %r8, %r6;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM90-NEXT: shr.u32 %r11, %r10, %r3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r11;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @and_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: and_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [and_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB37_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB37_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: and_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB38_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: and.b32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB38_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: and_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [and_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB39_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB39_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: and_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB40_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: and.b32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB40_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: and_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB41_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: and.b32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: and.b32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: and.b32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB41_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @and_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: and_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @and_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: and_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @and_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: and_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [and_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @and_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: and_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @and_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: and_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @and_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: and_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [and_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [and_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: nand_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB48_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, %r4;
+; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB48_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: nand_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [nand_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB49_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB49_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB50_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB50_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: nand_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [nand_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB51_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB51_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: nand_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB52_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB52_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: nand_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r16}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd4;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd3;
+; SM90-NEXT: $L__BB53_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r13;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r14;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r15;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r16;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r16, %r4;
+; SM90-NEXT: and.b32 %r6, %r15, %r3;
+; SM90-NEXT: and.b32 %r7, %r14, %r2;
+; SM90-NEXT: and.b32 %r8, %r13, %r1;
+; SM90-NEXT: xor.b32 %r9, %r8, -1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: xor.b32 %r10, %r7, -1;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: xor.b32 %r11, %r6, -1;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: xor.b32 %r12, %r5, -1;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r16}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd22;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB53_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB54_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB54_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB55_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r12, %r4;
+; SM90-NEXT: and.b32 %r6, %r11, %r3;
+; SM90-NEXT: and.b32 %r7, %r10, %r2;
+; SM90-NEXT: and.b32 %r8, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB55_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: nand_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<54>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB56_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r20;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r9, %r20, %r4;
+; SM90-NEXT: and.b32 %r10, %r19, %r3;
+; SM90-NEXT: and.b32 %r11, %r18, %r2;
+; SM90-NEXT: and.b32 %r12, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r12;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r10;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r9;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd24;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB56_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd28, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd29, %rd30}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd30;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
+; SM90-NEXT: $L__BB56_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
+; SM90-NEXT: shl.b64 %rd33, %rd32, 32;
+; SM90-NEXT: or.b64 %rd34, %rd31, %rd33;
+; SM90-NEXT: cvt.u64.u32 %rd35, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd36, %r24;
+; SM90-NEXT: shl.b64 %rd37, %rd36, 32;
+; SM90-NEXT: or.b64 %rd38, %rd35, %rd37;
+; SM90-NEXT: and.b32 %r13, %r24, %r8;
+; SM90-NEXT: and.b32 %r14, %r23, %r7;
+; SM90-NEXT: and.b32 %r15, %r22, %r6;
+; SM90-NEXT: and.b32 %r16, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd39, %r16;
+; SM90-NEXT: cvt.u64.u32 %rd40, %r15;
+; SM90-NEXT: shl.b64 %rd41, %rd40, 32;
+; SM90-NEXT: or.b64 %rd42, %rd39, %rd41;
+; SM90-NEXT: not.b64 %rd43, %rd42;
+; SM90-NEXT: cvt.u64.u32 %rd44, %r14;
+; SM90-NEXT: cvt.u64.u32 %rd45, %r13;
+; SM90-NEXT: shl.b64 %rd46, %rd45, 32;
+; SM90-NEXT: or.b64 %rd47, %rd44, %rd46;
+; SM90-NEXT: not.b64 %rd48, %rd47;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd38};
+; SM90-NEXT: mov.b128 swap, {%rd43, %rd48};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd49, %rd50}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd51, %rd50, %rd38;
+; SM90-NEXT: xor.b64 %rd52, %rd49, %rd34;
+; SM90-NEXT: or.b64 %rd53, %rd52, %rd51;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd50;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd50;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
+; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
+; SM90-NEXT: @%p2 bra $L__BB56_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: nand_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<16>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [nand_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd14, %rd15}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB57_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b64 %rd7, %rd15, %rd5;
+; SM90-NEXT: and.b64 %rd8, %rd14, %rd4;
+; SM90-NEXT: not.b64 %rd9, %rd8;
+; SM90-NEXT: not.b64 %rd10, %rd7;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd14, %rd15};
+; SM90-NEXT: mov.b128 swap, {%rd9, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd11, %rd2, %rd15;
+; SM90-NEXT: xor.b64 %rd12, %rd1, %rd14;
+; SM90-NEXT: or.b64 %rd13, %rd12, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd13, 0;
+; SM90-NEXT: mov.b64 %rd14, %rd1;
+; SM90-NEXT: mov.b64 %rd15, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB57_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: nand_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<30>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [nand_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd26, %rd27}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB58_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b64 %rd11, %rd27, %rd2;
+; SM90-NEXT: and.b64 %rd12, %rd26, %rd1;
+; SM90-NEXT: not.b64 %rd13, %rd12;
+; SM90-NEXT: not.b64 %rd14, %rd11;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd27};
+; SM90-NEXT: mov.b128 swap, {%rd13, %rd14};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd15, %rd6, %rd27;
+; SM90-NEXT: xor.b64 %rd16, %rd5, %rd26;
+; SM90-NEXT: or.b64 %rd17, %rd16, %rd15;
+; SM90-NEXT: setp.ne.b64 %p1, %rd17, 0;
+; SM90-NEXT: mov.b64 %rd26, %rd5;
+; SM90-NEXT: mov.b64 %rd27, %rd6;
+; SM90-NEXT: @%p1 bra $L__BB58_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd28, %rd29}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB58_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b64 %rd19, %rd29, %rd4;
+; SM90-NEXT: and.b64 %rd20, %rd28, %rd3;
+; SM90-NEXT: not.b64 %rd21, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd29};
+; SM90-NEXT: mov.b128 swap, {%rd21, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd8, %rd29;
+; SM90-NEXT: xor.b64 %rd24, %rd7, %rd28;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p2, %rd25, 0;
+; SM90-NEXT: mov.b64 %rd28, %rd7;
+; SM90-NEXT: mov.b64 %rd29, %rd8;
+; SM90-NEXT: @%p2 bra $L__BB58_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: nand_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<5>;
+; SM90-NEXT: .reg .b64 %rd<58>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [nand_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [nand_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [nand_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB59_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd51;
+; SM90-NEXT: mov.b64 %rd9, %rd50;
+; SM90-NEXT: and.b64 %rd19, %rd10, %rd6;
+; SM90-NEXT: and.b64 %rd20, %rd9, %rd5;
+; SM90-NEXT: not.b64 %rd21, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd21, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd51, %rd10;
+; SM90-NEXT: xor.b64 %rd24, %rd50, %rd9;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB59_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB59_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd53;
+; SM90-NEXT: mov.b64 %rd11, %rd52;
+; SM90-NEXT: and.b64 %rd27, %rd12, %rd8;
+; SM90-NEXT: and.b64 %rd28, %rd11, %rd7;
+; SM90-NEXT: not.b64 %rd29, %rd28;
+; SM90-NEXT: not.b64 %rd30, %rd27;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd29, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd31, %rd53, %rd12;
+; SM90-NEXT: xor.b64 %rd32, %rd52, %rd11;
+; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
+; SM90-NEXT: setp.ne.b64 %p2, %rd33, 0;
+; SM90-NEXT: @%p2 bra $L__BB59_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd34, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd34};
+; SM90-NEXT: mov.b128 swap, {%rd34, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd54, %rd55}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB59_5: // %atomicrmw.start16
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b64 %rd35, %rd55, %rd2;
+; SM90-NEXT: and.b64 %rd36, %rd54, %rd1;
+; SM90-NEXT: not.b64 %rd37, %rd36;
+; SM90-NEXT: not.b64 %rd38, %rd35;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd54, %rd55};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd14, %rd55;
+; SM90-NEXT: xor.b64 %rd40, %rd13, %rd54;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p3, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd54, %rd13;
+; SM90-NEXT: mov.b64 %rd55, %rd14;
+; SM90-NEXT: @%p3 bra $L__BB59_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end15
+; SM90-NEXT: mov.b64 %rd42, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd42, %rd42};
+; SM90-NEXT: mov.b128 swap, {%rd42, %rd42};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd56, %rd57}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB59_7: // %atomicrmw.start22
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b64 %rd43, %rd57, %rd4;
+; SM90-NEXT: and.b64 %rd44, %rd56, %rd3;
+; SM90-NEXT: not.b64 %rd45, %rd44;
+; SM90-NEXT: not.b64 %rd46, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd56, %rd57};
+; SM90-NEXT: mov.b128 swap, {%rd45, %rd46};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd16, %rd57;
+; SM90-NEXT: xor.b64 %rd48, %rd15, %rd56;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: setp.ne.b64 %p4, %rd49, 0;
+; SM90-NEXT: mov.b64 %rd56, %rd15;
+; SM90-NEXT: mov.b64 %rd57, %rd16;
+; SM90-NEXT: @%p4 bra $L__BB59_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end21
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @or_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: or_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<8>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM90-NEXT: and.b32 %r2, %r1, 3;
+; SM90-NEXT: shl.b32 %r3, %r2, 3;
+; SM90-NEXT: ld.param.b16 %r4, [or_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r5, %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM90-NEXT: shr.u32 %r7, %r6, %r3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r7;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @or_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: or_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [or_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: or.b32 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB61_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: or_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: or.b32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: or.b32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB62_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: or_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [or_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: or.b32 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB63_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: or_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: or.b32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: or.b32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB64_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: or_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: or.b32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: or.b32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: or.b32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: or.b32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB65_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @or_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: or_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @or_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: or_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @or_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: or_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [or_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @or_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: or_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @or_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: or_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @or_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: or_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [or_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [or_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @xor_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: xor_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<8>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM90-NEXT: and.b32 %r2, %r1, 3;
+; SM90-NEXT: shl.b32 %r3, %r2, 3;
+; SM90-NEXT: ld.param.b16 %r4, [xor_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r5, %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM90-NEXT: shr.u32 %r7, %r6, %r3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r7;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @xor_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: xor_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [xor_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: xor.b32 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB73_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: xor_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: xor.b32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: xor.b32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB74_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: xor_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [xor_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: xor.b32 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB75_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: xor_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: xor.b32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: xor.b32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB76_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: xor_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: xor.b32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: xor.b32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: xor.b32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: xor.b32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB77_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @xor_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: xor_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @xor_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: xor_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @xor_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: xor_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xor_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @xor_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: xor_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @xor_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: xor_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @xor_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: xor_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xor_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xor_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: max_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<14>;
+; SM90-NEXT: .reg .b32 %r<20>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [max_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM90-NEXT: cvt.s16.s8 %rs7, %rs6;
+; SM90-NEXT: cvt.s16.s8 %rs8, %rs5;
+; SM90-NEXT: mov.b32 %r12, {%rs8, %rs7};
+; SM90-NEXT: $L__BB84_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r19, %r1;
+; SM90-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM90-NEXT: bfe.s32 %r10, %r9, 8, 8;
+; SM90-NEXT: cvt.s8.s32 %rs4, %r10;
+; SM90-NEXT: mov.b32 %r11, {%rs3, %rs4};
+; SM90-NEXT: max.s16x2 %r13, %r11, %r12;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r13;
+; SM90-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM90-NEXT: and.b16 %rs12, %rs9, 255;
+; SM90-NEXT: or.b16 %rs13, %rs12, %rs11;
+; SM90-NEXT: cvt.u32.u16 %r14, %rs13;
+; SM90-NEXT: shl.b32 %r15, %r14, %r1;
+; SM90-NEXT: and.b32 %r16, %r19, %r2;
+; SM90-NEXT: or.b32 %r17, %r16, %r15;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r19;
+; SM90-NEXT: mov.b32 %r19, %r3;
+; SM90-NEXT: @%p1 bra $L__BB84_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r18, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r18;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: max_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b32 %r<27>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [max_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM90-NEXT: $L__BB85_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
+; SM90-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
+; SM90-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
+; SM90-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
+; SM90-NEXT: setp.gt.s32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r15, %r26, 0, 0x7773U;
+; SM90-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; SM90-NEXT: prmt.b32 %r17, %r26, 0, 0x7772U;
+; SM90-NEXT: selp.b32 %r18, %r17, %r13, %p3;
+; SM90-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
+; SM90-NEXT: prmt.b32 %r20, %r26, 0, 0x7771U;
+; SM90-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; SM90-NEXT: prmt.b32 %r22, %r26, 0, 0x7770U;
+; SM90-NEXT: selp.b32 %r23, %r22, %r11, %p1;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r26;
+; SM90-NEXT: mov.b32 %r26, %r1;
+; SM90-NEXT: @%p5 bra $L__BB85_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: max_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<51>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r50}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM90-NEXT: $L__BB86_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r49;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
+; SM90-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
+; SM90-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM90-NEXT: prmt.b32 %r8, %r50, 0, 0xaaa2U;
+; SM90-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM90-NEXT: prmt.b32 %r10, %r50, 0, 0xbbb3U;
+; SM90-NEXT: setp.gt.s32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r1, 0, 0x8880U;
+; SM90-NEXT: prmt.b32 %r12, %r49, 0, 0x8880U;
+; SM90-NEXT: setp.gt.s32 %p5, %r12, %r11;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x9991U;
+; SM90-NEXT: prmt.b32 %r14, %r49, 0, 0x9991U;
+; SM90-NEXT: setp.gt.s32 %p6, %r14, %r13;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0xaaa2U;
+; SM90-NEXT: prmt.b32 %r16, %r49, 0, 0xaaa2U;
+; SM90-NEXT: setp.gt.s32 %p7, %r16, %r15;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0xbbb3U;
+; SM90-NEXT: prmt.b32 %r18, %r49, 0, 0xbbb3U;
+; SM90-NEXT: setp.gt.s32 %p8, %r18, %r17;
+; SM90-NEXT: prmt.b32 %r19, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r19;
+; SM90-NEXT: prmt.b32 %r20, %r50, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r20;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r50, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r22;
+; SM90-NEXT: prmt.b32 %r23, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r23;
+; SM90-NEXT: prmt.b32 %r24, %r50, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r24;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r50, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r26;
+; SM90-NEXT: prmt.b32 %r27, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r27;
+; SM90-NEXT: prmt.b32 %r28, %r49, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r29;
+; SM90-NEXT: prmt.b32 %r30, %r49, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs12, %r30;
+; SM90-NEXT: prmt.b32 %r31, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r31;
+; SM90-NEXT: prmt.b32 %r32, %r49, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r32;
+; SM90-NEXT: prmt.b32 %r33, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r33;
+; SM90-NEXT: prmt.b32 %r34, %r49, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r34;
+; SM90-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM90-NEXT: cvt.u32.u16 %r35, %rs17;
+; SM90-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM90-NEXT: cvt.u32.u16 %r36, %rs18;
+; SM90-NEXT: prmt.b32 %r37, %r36, %r35, 0x3340U;
+; SM90-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM90-NEXT: cvt.u32.u16 %r38, %rs19;
+; SM90-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM90-NEXT: cvt.u32.u16 %r39, %rs20;
+; SM90-NEXT: prmt.b32 %r40, %r39, %r38, 0x3340U;
+; SM90-NEXT: prmt.b32 %r41, %r40, %r37, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM90-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM90-NEXT: cvt.u32.u16 %r42, %rs21;
+; SM90-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM90-NEXT: cvt.u32.u16 %r43, %rs22;
+; SM90-NEXT: prmt.b32 %r44, %r43, %r42, 0x3340U;
+; SM90-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM90-NEXT: cvt.u32.u16 %r45, %rs23;
+; SM90-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM90-NEXT: cvt.u32.u16 %r46, %rs24;
+; SM90-NEXT: prmt.b32 %r47, %r46, %r45, 0x3340U;
+; SM90-NEXT: prmt.b32 %r48, %r47, %r44, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r48;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r50}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB86_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: max_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [max_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB87_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.s16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB87_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: max_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB88_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: max.s16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM90-NEXT: max.s16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB88_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: max_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB89_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: max.s16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r5;
+; SM90-NEXT: max.s16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r6;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: max.s16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM90-NEXT: max.s16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB89_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @max_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: max_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @max_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: max_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @max_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: max_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [max_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @max_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: max_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @max_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: max_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @max_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: max_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [max_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [max_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: min_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<14>;
+; SM90-NEXT: .reg .b32 %r<20>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [min_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM90-NEXT: cvt.s16.s8 %rs7, %rs6;
+; SM90-NEXT: cvt.s16.s8 %rs8, %rs5;
+; SM90-NEXT: mov.b32 %r12, {%rs8, %rs7};
+; SM90-NEXT: $L__BB96_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r19, %r1;
+; SM90-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM90-NEXT: bfe.s32 %r10, %r9, 8, 8;
+; SM90-NEXT: cvt.s8.s32 %rs4, %r10;
+; SM90-NEXT: mov.b32 %r11, {%rs3, %rs4};
+; SM90-NEXT: min.s16x2 %r13, %r11, %r12;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r13;
+; SM90-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM90-NEXT: and.b16 %rs12, %rs9, 255;
+; SM90-NEXT: or.b16 %rs13, %rs12, %rs11;
+; SM90-NEXT: cvt.u32.u16 %r14, %rs13;
+; SM90-NEXT: shl.b32 %r15, %r14, %r1;
+; SM90-NEXT: and.b32 %r16, %r19, %r2;
+; SM90-NEXT: or.b32 %r17, %r16, %r15;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r19;
+; SM90-NEXT: mov.b32 %r19, %r3;
+; SM90-NEXT: @%p1 bra $L__BB96_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r18, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r18;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: min_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b32 %r<27>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [min_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM90-NEXT: $L__BB97_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
+; SM90-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
+; SM90-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
+; SM90-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
+; SM90-NEXT: setp.le.s32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r15, %r26, 0, 0x7773U;
+; SM90-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; SM90-NEXT: prmt.b32 %r17, %r26, 0, 0x7772U;
+; SM90-NEXT: selp.b32 %r18, %r17, %r13, %p3;
+; SM90-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
+; SM90-NEXT: prmt.b32 %r20, %r26, 0, 0x7771U;
+; SM90-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; SM90-NEXT: prmt.b32 %r22, %r26, 0, 0x7770U;
+; SM90-NEXT: selp.b32 %r23, %r22, %r11, %p1;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r26;
+; SM90-NEXT: mov.b32 %r26, %r1;
+; SM90-NEXT: @%p5 bra $L__BB97_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: min_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<51>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r50}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM90-NEXT: $L__BB98_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r49;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
+; SM90-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
+; SM90-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM90-NEXT: prmt.b32 %r8, %r50, 0, 0xaaa2U;
+; SM90-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM90-NEXT: prmt.b32 %r10, %r50, 0, 0xbbb3U;
+; SM90-NEXT: setp.le.s32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r1, 0, 0x8880U;
+; SM90-NEXT: prmt.b32 %r12, %r49, 0, 0x8880U;
+; SM90-NEXT: setp.le.s32 %p5, %r12, %r11;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x9991U;
+; SM90-NEXT: prmt.b32 %r14, %r49, 0, 0x9991U;
+; SM90-NEXT: setp.le.s32 %p6, %r14, %r13;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0xaaa2U;
+; SM90-NEXT: prmt.b32 %r16, %r49, 0, 0xaaa2U;
+; SM90-NEXT: setp.le.s32 %p7, %r16, %r15;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0xbbb3U;
+; SM90-NEXT: prmt.b32 %r18, %r49, 0, 0xbbb3U;
+; SM90-NEXT: setp.le.s32 %p8, %r18, %r17;
+; SM90-NEXT: prmt.b32 %r19, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r19;
+; SM90-NEXT: prmt.b32 %r20, %r50, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r20;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r50, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r22;
+; SM90-NEXT: prmt.b32 %r23, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r23;
+; SM90-NEXT: prmt.b32 %r24, %r50, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r24;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r50, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r26;
+; SM90-NEXT: prmt.b32 %r27, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r27;
+; SM90-NEXT: prmt.b32 %r28, %r49, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r29;
+; SM90-NEXT: prmt.b32 %r30, %r49, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs12, %r30;
+; SM90-NEXT: prmt.b32 %r31, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r31;
+; SM90-NEXT: prmt.b32 %r32, %r49, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r32;
+; SM90-NEXT: prmt.b32 %r33, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r33;
+; SM90-NEXT: prmt.b32 %r34, %r49, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r34;
+; SM90-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM90-NEXT: cvt.u32.u16 %r35, %rs17;
+; SM90-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM90-NEXT: cvt.u32.u16 %r36, %rs18;
+; SM90-NEXT: prmt.b32 %r37, %r36, %r35, 0x3340U;
+; SM90-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM90-NEXT: cvt.u32.u16 %r38, %rs19;
+; SM90-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM90-NEXT: cvt.u32.u16 %r39, %rs20;
+; SM90-NEXT: prmt.b32 %r40, %r39, %r38, 0x3340U;
+; SM90-NEXT: prmt.b32 %r41, %r40, %r37, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM90-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM90-NEXT: cvt.u32.u16 %r42, %rs21;
+; SM90-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM90-NEXT: cvt.u32.u16 %r43, %rs22;
+; SM90-NEXT: prmt.b32 %r44, %r43, %r42, 0x3340U;
+; SM90-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM90-NEXT: cvt.u32.u16 %r45, %rs23;
+; SM90-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM90-NEXT: cvt.u32.u16 %r46, %rs24;
+; SM90-NEXT: prmt.b32 %r47, %r46, %r45, 0x3340U;
+; SM90-NEXT: prmt.b32 %r48, %r47, %r44, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r48;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r50}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB98_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: min_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [min_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB99_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.s16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB99_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: min_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB100_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: min.s16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM90-NEXT: min.s16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB100_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: min_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB101_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: min.s16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r5;
+; SM90-NEXT: min.s16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r6;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: min.s16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM90-NEXT: min.s16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB101_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @min_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: min_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @min_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: min_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @min_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: min_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [min_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @min_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: min_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @min_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: min_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @min_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: min_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [min_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [min_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: umax_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umax_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB108_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, 16711935;
+; SM90-NEXT: max.u16x2 %r12, %r11, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB108_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: umax_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [umax_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM90-NEXT: $L__BB109_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
+; SM90-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
+; SM90-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
+; SM90-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
+; SM90-NEXT: setp.gt.u32 %p4, %r10, %r9;
+; SM90-NEXT: selp.b32 %r11, %r10, %r9, %p4;
+; SM90-NEXT: selp.b32 %r12, %r8, %r7, %p3;
+; SM90-NEXT: prmt.b32 %r13, %r12, %r11, 0x3340U;
+; SM90-NEXT: selp.b32 %r14, %r6, %r5, %p2;
+; SM90-NEXT: selp.b32 %r15, %r4, %r3, %p1;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r14, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r13, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p5 bra $L__BB109_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: umax_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM90-NEXT: $L__BB110_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM90-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM90-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM90-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM90-NEXT: setp.gt.u32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM90-NEXT: setp.gt.u32 %p5, %r12, %r11;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM90-NEXT: setp.gt.u32 %p6, %r14, %r13;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM90-NEXT: setp.gt.u32 %p7, %r16, %r15;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM90-NEXT: setp.gt.u32 %p8, %r18, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r8;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r9;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r10;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r11;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r12;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r13;
+; SM90-NEXT: cvt.u16.u32 %rs12, %r14;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r15;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r16;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r18;
+; SM90-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM90-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM90-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM90-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM90-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM90-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM90-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM90-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM90-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM90-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM90-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM90-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM90-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB110_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: umax_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [umax_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB111_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB111_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: umax_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB112_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: max.u16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM90-NEXT: max.u16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB112_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: umax_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB113_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: max.u16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r5;
+; SM90-NEXT: max.u16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r6;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: max.u16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM90-NEXT: max.u16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB113_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @umax_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: umax_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @umax_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: umax_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @umax_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: umax_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umax_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @umax_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: umax_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @umax_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: umax_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @umax_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: umax_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umax_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umax_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: umin_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umin_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB120_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, 16711935;
+; SM90-NEXT: min.u16x2 %r12, %r11, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB120_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: umin_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [umin_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM90-NEXT: $L__BB121_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
+; SM90-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
+; SM90-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
+; SM90-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
+; SM90-NEXT: setp.le.u32 %p4, %r10, %r9;
+; SM90-NEXT: selp.b32 %r11, %r10, %r9, %p4;
+; SM90-NEXT: selp.b32 %r12, %r8, %r7, %p3;
+; SM90-NEXT: prmt.b32 %r13, %r12, %r11, 0x3340U;
+; SM90-NEXT: selp.b32 %r14, %r6, %r5, %p2;
+; SM90-NEXT: selp.b32 %r15, %r4, %r3, %p1;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r14, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r13, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p5 bra $L__BB121_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: umin_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM90-NEXT: $L__BB122_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM90-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM90-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM90-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM90-NEXT: setp.le.u32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM90-NEXT: setp.le.u32 %p5, %r12, %r11;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM90-NEXT: setp.le.u32 %p6, %r14, %r13;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM90-NEXT: setp.le.u32 %p7, %r16, %r15;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM90-NEXT: setp.le.u32 %p8, %r18, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r8;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r9;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r10;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r11;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r12;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r13;
+; SM90-NEXT: cvt.u16.u32 %rs12, %r14;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r15;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r16;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r18;
+; SM90-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM90-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM90-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM90-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM90-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM90-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM90-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM90-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM90-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM90-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM90-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM90-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM90-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB122_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: umin_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [umin_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB123_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.u16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB123_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: umin_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB124_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: min.u16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM90-NEXT: min.u16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB124_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: umin_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB125_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: min.u16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r5;
+; SM90-NEXT: min.u16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r6;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: min.u16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM90-NEXT: min.u16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB125_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @umin_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: umin_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @umin_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: umin_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @umin_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: umin_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umin_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @umin_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: umin_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @umin_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: umin_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @umin_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: umin_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umin_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umin_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b16 %rs<15>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [uinc_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: mov.b32 {%rs6, %rs7}, %r4;
+; SM90-NEXT: $L__BB132_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b16 %rs5, %rs3, 255;
+; SM90-NEXT: mov.b32 %r11, 65537;
+; SM90-NEXT: add.s16x2 %r12, %r10, %r11;
+; SM90-NEXT: setp.ge.u16 %p1, %rs4, %rs7;
+; SM90-NEXT: setp.ge.u16 %p2, %rs5, %rs6;
+; SM90-NEXT: mov.b32 {%rs8, %rs9}, %r12;
+; SM90-NEXT: selp.b16 %rs10, 0, %rs8, %p2;
+; SM90-NEXT: selp.b16 %rs11, 0, %rs9, %p1;
+; SM90-NEXT: shl.b16 %rs12, %rs11, 8;
+; SM90-NEXT: and.b16 %rs13, %rs10, 255;
+; SM90-NEXT: or.b16 %rs14, %rs13, %rs12;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs14;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p3, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p3 bra $L__BB132_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<23>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r22, [%rd1];
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
+; SM90-NEXT: $L__BB133_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r3, %r22, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: add.s16 %rs2, %rs1, 1;
+; SM90-NEXT: prmt.b32 %r4, %r22, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM90-NEXT: add.s16 %rs4, %rs3, 1;
+; SM90-NEXT: prmt.b32 %r5, %r22, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM90-NEXT: add.s16 %rs6, %rs5, 1;
+; SM90-NEXT: prmt.b32 %r6, %r22, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM90-NEXT: add.s16 %rs8, %rs7, 1;
+; SM90-NEXT: setp.ge.u32 %p1, %r3, %r7;
+; SM90-NEXT: setp.ge.u32 %p2, %r4, %r8;
+; SM90-NEXT: setp.ge.u32 %p3, %r5, %r9;
+; SM90-NEXT: setp.ge.u32 %p4, %r6, %r10;
+; SM90-NEXT: cvt.u32.u16 %r11, %rs8;
+; SM90-NEXT: selp.b32 %r12, 0, %r11, %p4;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs6;
+; SM90-NEXT: selp.b32 %r14, 0, %r13, %p3;
+; SM90-NEXT: prmt.b32 %r15, %r14, %r12, 0x3340U;
+; SM90-NEXT: cvt.u32.u16 %r16, %rs4;
+; SM90-NEXT: selp.b32 %r17, 0, %r16, %p2;
+; SM90-NEXT: cvt.u32.u16 %r18, %rs2;
+; SM90-NEXT: selp.b32 %r19, 0, %r18, %p1;
+; SM90-NEXT: prmt.b32 %r20, %r19, %r17, 0x3340U;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r15, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r22;
+; SM90-NEXT: mov.b32 %r22, %r1;
+; SM90-NEXT: @%p5 bra $L__BB133_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM90-NEXT: $L__BB134_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r3, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: add.s16 %rs2, %rs1, 1;
+; SM90-NEXT: prmt.b32 %r4, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM90-NEXT: add.s16 %rs4, %rs3, 1;
+; SM90-NEXT: prmt.b32 %r5, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM90-NEXT: add.s16 %rs6, %rs5, 1;
+; SM90-NEXT: prmt.b32 %r6, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM90-NEXT: add.s16 %rs8, %rs7, 1;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r7;
+; SM90-NEXT: add.s16 %rs10, %rs9, 1;
+; SM90-NEXT: prmt.b32 %r8, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r8;
+; SM90-NEXT: add.s16 %rs12, %rs11, 1;
+; SM90-NEXT: prmt.b32 %r9, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r9;
+; SM90-NEXT: add.s16 %rs14, %rs13, 1;
+; SM90-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r10;
+; SM90-NEXT: add.s16 %rs16, %rs15, 1;
+; SM90-NEXT: setp.ge.u32 %p1, %r3, %r11;
+; SM90-NEXT: setp.ge.u32 %p2, %r4, %r12;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM90-NEXT: setp.ge.u32 %p3, %r5, %r13;
+; SM90-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM90-NEXT: setp.ge.u32 %p4, %r6, %r14;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0x7770U;
+; SM90-NEXT: setp.ge.u32 %p5, %r7, %r15;
+; SM90-NEXT: prmt.b32 %r16, %r1, 0, 0x7771U;
+; SM90-NEXT: setp.ge.u32 %p6, %r8, %r16;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0x7772U;
+; SM90-NEXT: setp.ge.u32 %p7, %r9, %r17;
+; SM90-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
+; SM90-NEXT: setp.ge.u32 %p8, %r10, %r18;
+; SM90-NEXT: selp.b16 %rs17, 0, %rs16, %p8;
+; SM90-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM90-NEXT: selp.b16 %rs18, 0, %rs14, %p7;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM90-NEXT: selp.b16 %rs19, 0, %rs12, %p6;
+; SM90-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM90-NEXT: selp.b16 %rs20, 0, %rs10, %p5;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM90-NEXT: selp.b16 %rs21, 0, %rs8, %p4;
+; SM90-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM90-NEXT: selp.b16 %rs22, 0, %rs6, %p3;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM90-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM90-NEXT: selp.b16 %rs23, 0, %rs4, %p2;
+; SM90-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM90-NEXT: selp.b16 %rs24, 0, %rs2, %p1;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB134_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r6, [%rd1];
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM90-NEXT: $L__BB135_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b32 %r3, 65537;
+; SM90-NEXT: add.s16x2 %r4, %r6, %r3;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r6;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM90-NEXT: selp.b16 %rs7, 0, %rs6, %p2;
+; SM90-NEXT: selp.b16 %rs8, 0, %rs5, %p1;
+; SM90-NEXT: mov.b32 %r5, {%rs8, %rs7};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r6, %r5;
+; SM90-NEXT: setp.ne.b32 %p3, %r1, %r6;
+; SM90-NEXT: mov.b32 %r6, %r1;
+; SM90-NEXT: @%p3 bra $L__BB135_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b16 %rs<17>;
+; SM90-NEXT: .reg .b32 %r<10>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r9}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; SM90-NEXT: $L__BB136_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r9;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: mov.b32 %r3, 65537;
+; SM90-NEXT: add.s16x2 %r4, %r9, %r3;
+; SM90-NEXT: add.s16x2 %r5, %r8, %r3;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r8;
+; SM90-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM90-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r5;
+; SM90-NEXT: selp.b16 %rs11, 0, %rs10, %p4;
+; SM90-NEXT: selp.b16 %rs12, 0, %rs9, %p3;
+; SM90-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM90-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM90-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM90-NEXT: selp.b16 %rs15, 0, %rs14, %p2;
+; SM90-NEXT: selp.b16 %rs16, 0, %rs13, %p1;
+; SM90-NEXT: mov.b32 %r7, {%rs16, %rs15};
+; SM90-NEXT: cvt.u64.u32 %rd8, %r7;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r9}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r8, %rd11;
+; SM90-NEXT: @%p5 bra $L__BB136_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r8, %r9};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<33>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r17}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r16, %rd4;
+; SM90-NEXT: mov.b64 {_, %r15}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r14, %rd3;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r3;
+; SM90-NEXT: $L__BB137_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r14;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r15;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r16;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r17;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: mov.b32 %r5, 65537;
+; SM90-NEXT: add.s16x2 %r6, %r17, %r5;
+; SM90-NEXT: add.s16x2 %r7, %r16, %r5;
+; SM90-NEXT: add.s16x2 %r8, %r15, %r5;
+; SM90-NEXT: add.s16x2 %r9, %r14, %r5;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r17;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r16;
+; SM90-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM90-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM90-NEXT: mov.b32 {%rs11, %rs12}, %r15;
+; SM90-NEXT: setp.ge.u16 %p5, %rs11, %rs9;
+; SM90-NEXT: setp.ge.u16 %p6, %rs12, %rs10;
+; SM90-NEXT: mov.b32 {%rs13, %rs14}, %r1;
+; SM90-NEXT: mov.b32 {%rs15, %rs16}, %r14;
+; SM90-NEXT: setp.ge.u16 %p7, %rs15, %rs13;
+; SM90-NEXT: setp.ge.u16 %p8, %rs16, %rs14;
+; SM90-NEXT: mov.b32 {%rs17, %rs18}, %r9;
+; SM90-NEXT: selp.b16 %rs19, 0, %rs18, %p8;
+; SM90-NEXT: selp.b16 %rs20, 0, %rs17, %p7;
+; SM90-NEXT: mov.b32 %r10, {%rs20, %rs19};
+; SM90-NEXT: cvt.u64.u32 %rd13, %r10;
+; SM90-NEXT: mov.b32 {%rs21, %rs22}, %r8;
+; SM90-NEXT: selp.b16 %rs23, 0, %rs22, %p6;
+; SM90-NEXT: selp.b16 %rs24, 0, %rs21, %p5;
+; SM90-NEXT: mov.b32 %r11, {%rs24, %rs23};
+; SM90-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: mov.b32 {%rs25, %rs26}, %r7;
+; SM90-NEXT: selp.b16 %rs27, 0, %rs26, %p4;
+; SM90-NEXT: selp.b16 %rs28, 0, %rs25, %p3;
+; SM90-NEXT: mov.b32 %r12, {%rs28, %rs27};
+; SM90-NEXT: cvt.u64.u32 %rd17, %r12;
+; SM90-NEXT: mov.b32 {%rs29, %rs30}, %r6;
+; SM90-NEXT: selp.b16 %rs31, 0, %rs30, %p2;
+; SM90-NEXT: selp.b16 %rs32, 0, %rs29, %p1;
+; SM90-NEXT: mov.b32 %r13, {%rs32, %rs31};
+; SM90-NEXT: cvt.u64.u32 %rd18, %r13;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r17}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r16, %rd22;
+; SM90-NEXT: mov.b64 {_, %r15}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r14, %rd21;
+; SM90-NEXT: setp.ne.b64 %p9, %rd25, 0;
+; SM90-NEXT: @%p9 bra $L__BB137_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r14, %r15, %r16, %r17};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @uinc_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @uinc_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @uinc_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b64 %rd<16>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd14, %rd15}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB141_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd7, %rd15, 1;
+; SM90-NEXT: add.s64 %rd8, %rd14, 1;
+; SM90-NEXT: setp.ge.u64 %p1, %rd15, %rd5;
+; SM90-NEXT: setp.ge.u64 %p2, %rd14, %rd4;
+; SM90-NEXT: selp.b64 %rd9, 0, %rd8, %p2;
+; SM90-NEXT: selp.b64 %rd10, 0, %rd7, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd14, %rd15};
+; SM90-NEXT: mov.b128 swap, {%rd9, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd11, %rd2, %rd15;
+; SM90-NEXT: xor.b64 %rd12, %rd1, %rd14;
+; SM90-NEXT: or.b64 %rd13, %rd12, %rd11;
+; SM90-NEXT: setp.ne.b64 %p3, %rd13, 0;
+; SM90-NEXT: mov.b64 %rd14, %rd1;
+; SM90-NEXT: mov.b64 %rd15, %rd2;
+; SM90-NEXT: @%p3 bra $L__BB141_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<7>;
+; SM90-NEXT: .reg .b64 %rd<30>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [uinc_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd26, %rd27}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB142_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd11, %rd27, 1;
+; SM90-NEXT: add.s64 %rd12, %rd26, 1;
+; SM90-NEXT: setp.ge.u64 %p1, %rd27, %rd2;
+; SM90-NEXT: setp.ge.u64 %p2, %rd26, %rd1;
+; SM90-NEXT: selp.b64 %rd13, 0, %rd12, %p2;
+; SM90-NEXT: selp.b64 %rd14, 0, %rd11, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd27};
+; SM90-NEXT: mov.b128 swap, {%rd13, %rd14};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd15, %rd6, %rd27;
+; SM90-NEXT: xor.b64 %rd16, %rd5, %rd26;
+; SM90-NEXT: or.b64 %rd17, %rd16, %rd15;
+; SM90-NEXT: setp.ne.b64 %p3, %rd17, 0;
+; SM90-NEXT: mov.b64 %rd26, %rd5;
+; SM90-NEXT: mov.b64 %rd27, %rd6;
+; SM90-NEXT: @%p3 bra $L__BB142_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd28, %rd29}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB142_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd19, %rd29, 1;
+; SM90-NEXT: add.s64 %rd20, %rd28, 1;
+; SM90-NEXT: setp.ge.u64 %p4, %rd29, %rd4;
+; SM90-NEXT: setp.ge.u64 %p5, %rd28, %rd3;
+; SM90-NEXT: selp.b64 %rd21, 0, %rd20, %p5;
+; SM90-NEXT: selp.b64 %rd22, 0, %rd19, %p4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd29};
+; SM90-NEXT: mov.b128 swap, {%rd21, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd8, %rd29;
+; SM90-NEXT: xor.b64 %rd24, %rd7, %rd28;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p6, %rd25, 0;
+; SM90-NEXT: mov.b64 %rd28, %rd7;
+; SM90-NEXT: mov.b64 %rd29, %rd8;
+; SM90-NEXT: @%p6 bra $L__BB142_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<13>;
+; SM90-NEXT: .reg .b64 %rd<58>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [uinc_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB143_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd51;
+; SM90-NEXT: mov.b64 %rd9, %rd50;
+; SM90-NEXT: add.s64 %rd19, %rd10, 1;
+; SM90-NEXT: add.s64 %rd20, %rd9, 1;
+; SM90-NEXT: setp.ge.u64 %p1, %rd10, %rd6;
+; SM90-NEXT: setp.ge.u64 %p2, %rd9, %rd5;
+; SM90-NEXT: selp.b64 %rd21, 0, %rd20, %p2;
+; SM90-NEXT: selp.b64 %rd22, 0, %rd19, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd21, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd51, %rd10;
+; SM90-NEXT: xor.b64 %rd24, %rd50, %rd9;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p3, %rd25, 0;
+; SM90-NEXT: @%p3 bra $L__BB143_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB143_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd53;
+; SM90-NEXT: mov.b64 %rd11, %rd52;
+; SM90-NEXT: add.s64 %rd27, %rd12, 1;
+; SM90-NEXT: add.s64 %rd28, %rd11, 1;
+; SM90-NEXT: setp.ge.u64 %p4, %rd12, %rd8;
+; SM90-NEXT: setp.ge.u64 %p5, %rd11, %rd7;
+; SM90-NEXT: selp.b64 %rd29, 0, %rd28, %p5;
+; SM90-NEXT: selp.b64 %rd30, 0, %rd27, %p4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd29, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd31, %rd53, %rd12;
+; SM90-NEXT: xor.b64 %rd32, %rd52, %rd11;
+; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
+; SM90-NEXT: setp.ne.b64 %p6, %rd33, 0;
+; SM90-NEXT: @%p6 bra $L__BB143_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd34, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd34};
+; SM90-NEXT: mov.b128 swap, {%rd34, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd54, %rd55}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB143_5: // %atomicrmw.start16
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd35, %rd55, 1;
+; SM90-NEXT: add.s64 %rd36, %rd54, 1;
+; SM90-NEXT: setp.ge.u64 %p7, %rd55, %rd2;
+; SM90-NEXT: setp.ge.u64 %p8, %rd54, %rd1;
+; SM90-NEXT: selp.b64 %rd37, 0, %rd36, %p8;
+; SM90-NEXT: selp.b64 %rd38, 0, %rd35, %p7;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd54, %rd55};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd14, %rd55;
+; SM90-NEXT: xor.b64 %rd40, %rd13, %rd54;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p9, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd54, %rd13;
+; SM90-NEXT: mov.b64 %rd55, %rd14;
+; SM90-NEXT: @%p9 bra $L__BB143_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end15
+; SM90-NEXT: mov.b64 %rd42, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd42, %rd42};
+; SM90-NEXT: mov.b128 swap, {%rd42, %rd42};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd56, %rd57}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB143_7: // %atomicrmw.start22
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd43, %rd57, 1;
+; SM90-NEXT: add.s64 %rd44, %rd56, 1;
+; SM90-NEXT: setp.ge.u64 %p10, %rd57, %rd4;
+; SM90-NEXT: setp.ge.u64 %p11, %rd56, %rd3;
+; SM90-NEXT: selp.b64 %rd45, 0, %rd44, %p11;
+; SM90-NEXT: selp.b64 %rd46, 0, %rd43, %p10;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd56, %rd57};
+; SM90-NEXT: mov.b128 swap, {%rd45, %rd46};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd16, %rd57;
+; SM90-NEXT: xor.b64 %rd48, %rd15, %rd56;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: setp.ne.b64 %p12, %rd49, 0;
+; SM90-NEXT: mov.b64 %rd56, %rd15;
+; SM90-NEXT: mov.b64 %rd57, %rd16;
+; SM90-NEXT: @%p12 bra $L__BB143_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end21
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b16 %rs<17>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [udec_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: mov.b32 {%rs8, %rs9}, %r4;
+; SM90-NEXT: $L__BB144_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r15, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: and.b16 %rs4, %rs3, 255;
+; SM90-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM90-NEXT: add.s16 %rs6, %rs5, -1;
+; SM90-NEXT: add.s16 %rs7, %rs3, -1;
+; SM90-NEXT: setp.eq.b16 %p1, %rs5, 0;
+; SM90-NEXT: setp.eq.b16 %p2, %rs4, 0;
+; SM90-NEXT: setp.gt.u16 %p3, %rs5, %rs9;
+; SM90-NEXT: setp.gt.u16 %p4, %rs4, %rs8;
+; SM90-NEXT: selp.b16 %rs10, %rs8, %rs7, %p4;
+; SM90-NEXT: selp.b16 %rs11, %rs8, %rs10, %p2;
+; SM90-NEXT: selp.b16 %rs12, %rs9, %rs6, %p3;
+; SM90-NEXT: selp.b16 %rs13, %rs9, %rs12, %p1;
+; SM90-NEXT: shl.b16 %rs14, %rs13, 8;
+; SM90-NEXT: and.b16 %rs15, %rs11, 255;
+; SM90-NEXT: or.b16 %rs16, %rs15, %rs14;
+; SM90-NEXT: cvt.u32.u16 %r10, %rs16;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p5, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p5 bra $L__BB144_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<27>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
+; SM90-NEXT: $L__BB145_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r3, %r26, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: add.s16 %rs2, %rs1, -1;
+; SM90-NEXT: prmt.b32 %r4, %r26, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM90-NEXT: add.s16 %rs4, %rs3, -1;
+; SM90-NEXT: prmt.b32 %r5, %r26, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM90-NEXT: add.s16 %rs6, %rs5, -1;
+; SM90-NEXT: prmt.b32 %r6, %r26, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM90-NEXT: add.s16 %rs8, %rs7, -1;
+; SM90-NEXT: setp.eq.b32 %p1, %r3, 0;
+; SM90-NEXT: setp.eq.b32 %p2, %r4, 0;
+; SM90-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM90-NEXT: setp.eq.b32 %p4, %r6, 0;
+; SM90-NEXT: setp.gt.u32 %p5, %r3, %r7;
+; SM90-NEXT: setp.gt.u32 %p6, %r4, %r8;
+; SM90-NEXT: setp.gt.u32 %p7, %r5, %r9;
+; SM90-NEXT: setp.gt.u32 %p8, %r6, %r10;
+; SM90-NEXT: cvt.u32.u16 %r11, %rs8;
+; SM90-NEXT: selp.b32 %r12, %r10, %r11, %p8;
+; SM90-NEXT: selp.b32 %r13, %r10, %r12, %p4;
+; SM90-NEXT: cvt.u32.u16 %r14, %rs6;
+; SM90-NEXT: selp.b32 %r15, %r9, %r14, %p7;
+; SM90-NEXT: selp.b32 %r16, %r9, %r15, %p3;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r13, 0x3340U;
+; SM90-NEXT: cvt.u32.u16 %r18, %rs4;
+; SM90-NEXT: selp.b32 %r19, %r8, %r18, %p6;
+; SM90-NEXT: selp.b32 %r20, %r8, %r19, %p2;
+; SM90-NEXT: cvt.u32.u16 %r21, %rs2;
+; SM90-NEXT: selp.b32 %r22, %r7, %r21, %p5;
+; SM90-NEXT: selp.b32 %r23, %r7, %r22, %p1;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r17, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM90-NEXT: setp.ne.b32 %p9, %r1, %r26;
+; SM90-NEXT: mov.b32 %r26, %r1;
+; SM90-NEXT: @%p9 bra $L__BB145_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<18>;
+; SM90-NEXT: .reg .b16 %rs<41>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r11;
+; SM90-NEXT: cvt.u16.u32 %rs18, %r12;
+; SM90-NEXT: $L__BB146_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r3, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: add.s16 %rs2, %rs1, -1;
+; SM90-NEXT: prmt.b32 %r4, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM90-NEXT: add.s16 %rs4, %rs3, -1;
+; SM90-NEXT: prmt.b32 %r5, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM90-NEXT: add.s16 %rs6, %rs5, -1;
+; SM90-NEXT: prmt.b32 %r6, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM90-NEXT: add.s16 %rs8, %rs7, -1;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r7;
+; SM90-NEXT: add.s16 %rs10, %rs9, -1;
+; SM90-NEXT: prmt.b32 %r8, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r8;
+; SM90-NEXT: add.s16 %rs12, %rs11, -1;
+; SM90-NEXT: prmt.b32 %r9, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r9;
+; SM90-NEXT: add.s16 %rs14, %rs13, -1;
+; SM90-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r10;
+; SM90-NEXT: add.s16 %rs16, %rs15, -1;
+; SM90-NEXT: setp.eq.b32 %p1, %r3, 0;
+; SM90-NEXT: setp.eq.b32 %p2, %r4, 0;
+; SM90-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM90-NEXT: setp.eq.b32 %p4, %r6, 0;
+; SM90-NEXT: setp.eq.b32 %p5, %r7, 0;
+; SM90-NEXT: setp.eq.b32 %p6, %r8, 0;
+; SM90-NEXT: setp.eq.b32 %p7, %r9, 0;
+; SM90-NEXT: setp.eq.b32 %p8, %r10, 0;
+; SM90-NEXT: setp.gt.u32 %p9, %r3, %r11;
+; SM90-NEXT: setp.gt.u32 %p10, %r4, %r12;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM90-NEXT: setp.gt.u32 %p11, %r5, %r13;
+; SM90-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM90-NEXT: setp.gt.u32 %p12, %r6, %r14;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0x7770U;
+; SM90-NEXT: setp.gt.u32 %p13, %r7, %r15;
+; SM90-NEXT: prmt.b32 %r16, %r1, 0, 0x7771U;
+; SM90-NEXT: setp.gt.u32 %p14, %r8, %r16;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0x7772U;
+; SM90-NEXT: setp.gt.u32 %p15, %r9, %r17;
+; SM90-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
+; SM90-NEXT: setp.gt.u32 %p16, %r10, %r18;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r13;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r14;
+; SM90-NEXT: cvt.u16.u32 %rs21, %r15;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r16;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs24, %r18;
+; SM90-NEXT: selp.b16 %rs25, %rs24, %rs16, %p16;
+; SM90-NEXT: selp.b16 %rs26, %rs24, %rs25, %p8;
+; SM90-NEXT: cvt.u32.u16 %r19, %rs26;
+; SM90-NEXT: selp.b16 %rs27, %rs23, %rs14, %p15;
+; SM90-NEXT: selp.b16 %rs28, %rs23, %rs27, %p7;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs28;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM90-NEXT: selp.b16 %rs29, %rs22, %rs12, %p14;
+; SM90-NEXT: selp.b16 %rs30, %rs22, %rs29, %p6;
+; SM90-NEXT: cvt.u32.u16 %r22, %rs30;
+; SM90-NEXT: selp.b16 %rs31, %rs21, %rs10, %p13;
+; SM90-NEXT: selp.b16 %rs32, %rs21, %rs31, %p5;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs32;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM90-NEXT: selp.b16 %rs33, %rs20, %rs8, %p12;
+; SM90-NEXT: selp.b16 %rs34, %rs20, %rs33, %p4;
+; SM90-NEXT: cvt.u32.u16 %r26, %rs34;
+; SM90-NEXT: selp.b16 %rs35, %rs19, %rs6, %p11;
+; SM90-NEXT: selp.b16 %rs36, %rs19, %rs35, %p3;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs36;
+; SM90-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM90-NEXT: selp.b16 %rs37, %rs18, %rs4, %p10;
+; SM90-NEXT: selp.b16 %rs38, %rs18, %rs37, %p2;
+; SM90-NEXT: cvt.u32.u16 %r29, %rs38;
+; SM90-NEXT: selp.b16 %rs39, %rs17, %rs2, %p9;
+; SM90-NEXT: selp.b16 %rs40, %rs17, %rs39, %p1;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs40;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p17, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p17 bra $L__BB146_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b16 %rs<11>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM90-NEXT: $L__BB147_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs1, -1;
+; SM90-NEXT: add.s16 %rs4, %rs2, -1;
+; SM90-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM90-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM90-NEXT: setp.gt.u16 %p3, %rs1, %rs5;
+; SM90-NEXT: setp.gt.u16 %p4, %rs2, %rs6;
+; SM90-NEXT: selp.b16 %rs7, %rs6, %rs4, %p4;
+; SM90-NEXT: selp.b16 %rs8, %rs6, %rs7, %p2;
+; SM90-NEXT: selp.b16 %rs9, %rs5, %rs3, %p3;
+; SM90-NEXT: selp.b16 %rs10, %rs5, %rs9, %p1;
+; SM90-NEXT: mov.b32 %r3, {%rs10, %rs8};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p5 bra $L__BB147_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<21>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM90-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM90-NEXT: $L__BB148_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r6;
+; SM90-NEXT: add.s16 %rs3, %rs1, -1;
+; SM90-NEXT: add.s16 %rs4, %rs2, -1;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r5;
+; SM90-NEXT: add.s16 %rs7, %rs5, -1;
+; SM90-NEXT: add.s16 %rs8, %rs6, -1;
+; SM90-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM90-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM90-NEXT: setp.eq.b16 %p3, %rs5, 0;
+; SM90-NEXT: setp.eq.b16 %p4, %rs6, 0;
+; SM90-NEXT: setp.gt.u16 %p5, %rs1, %rs9;
+; SM90-NEXT: setp.gt.u16 %p6, %rs2, %rs10;
+; SM90-NEXT: setp.gt.u16 %p7, %rs5, %rs11;
+; SM90-NEXT: setp.gt.u16 %p8, %rs6, %rs12;
+; SM90-NEXT: selp.b16 %rs13, %rs12, %rs8, %p8;
+; SM90-NEXT: selp.b16 %rs14, %rs12, %rs13, %p4;
+; SM90-NEXT: selp.b16 %rs15, %rs11, %rs7, %p7;
+; SM90-NEXT: selp.b16 %rs16, %rs11, %rs15, %p3;
+; SM90-NEXT: mov.b32 %r3, {%rs16, %rs14};
+; SM90-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM90-NEXT: selp.b16 %rs17, %rs10, %rs4, %p6;
+; SM90-NEXT: selp.b16 %rs18, %rs10, %rs17, %p2;
+; SM90-NEXT: selp.b16 %rs19, %rs9, %rs3, %p5;
+; SM90-NEXT: selp.b16 %rs20, %rs9, %rs19, %p1;
+; SM90-NEXT: mov.b32 %r4, {%rs20, %rs18};
+; SM90-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB148_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<18>;
+; SM90-NEXT: .reg .b16 %rs<41>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: mov.b32 {%rs17, %rs18}, %r4;
+; SM90-NEXT: mov.b32 {%rs19, %rs20}, %r3;
+; SM90-NEXT: $L__BB149_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r12;
+; SM90-NEXT: add.s16 %rs3, %rs1, -1;
+; SM90-NEXT: add.s16 %rs4, %rs2, -1;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: add.s16 %rs7, %rs5, -1;
+; SM90-NEXT: add.s16 %rs8, %rs6, -1;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM90-NEXT: add.s16 %rs11, %rs9, -1;
+; SM90-NEXT: add.s16 %rs12, %rs10, -1;
+; SM90-NEXT: mov.b32 {%rs13, %rs14}, %r9;
+; SM90-NEXT: add.s16 %rs15, %rs13, -1;
+; SM90-NEXT: add.s16 %rs16, %rs14, -1;
+; SM90-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM90-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM90-NEXT: setp.eq.b16 %p3, %rs5, 0;
+; SM90-NEXT: setp.eq.b16 %p4, %rs6, 0;
+; SM90-NEXT: setp.eq.b16 %p5, %rs9, 0;
+; SM90-NEXT: setp.eq.b16 %p6, %rs10, 0;
+; SM90-NEXT: setp.eq.b16 %p7, %rs13, 0;
+; SM90-NEXT: setp.eq.b16 %p8, %rs14, 0;
+; SM90-NEXT: setp.gt.u16 %p9, %rs1, %rs17;
+; SM90-NEXT: setp.gt.u16 %p10, %rs2, %rs18;
+; SM90-NEXT: setp.gt.u16 %p11, %rs5, %rs19;
+; SM90-NEXT: setp.gt.u16 %p12, %rs6, %rs20;
+; SM90-NEXT: mov.b32 {%rs21, %rs22}, %r2;
+; SM90-NEXT: setp.gt.u16 %p13, %rs9, %rs21;
+; SM90-NEXT: setp.gt.u16 %p14, %rs10, %rs22;
+; SM90-NEXT: mov.b32 {%rs23, %rs24}, %r1;
+; SM90-NEXT: setp.gt.u16 %p15, %rs13, %rs23;
+; SM90-NEXT: setp.gt.u16 %p16, %rs14, %rs24;
+; SM90-NEXT: selp.b16 %rs25, %rs24, %rs16, %p16;
+; SM90-NEXT: selp.b16 %rs26, %rs24, %rs25, %p8;
+; SM90-NEXT: selp.b16 %rs27, %rs23, %rs15, %p15;
+; SM90-NEXT: selp.b16 %rs28, %rs23, %rs27, %p7;
+; SM90-NEXT: mov.b32 %r5, {%rs28, %rs26};
+; SM90-NEXT: cvt.u64.u32 %rd13, %r5;
+; SM90-NEXT: selp.b16 %rs29, %rs22, %rs12, %p14;
+; SM90-NEXT: selp.b16 %rs30, %rs22, %rs29, %p6;
+; SM90-NEXT: selp.b16 %rs31, %rs21, %rs11, %p13;
+; SM90-NEXT: selp.b16 %rs32, %rs21, %rs31, %p5;
+; SM90-NEXT: mov.b32 %r6, {%rs32, %rs30};
+; SM90-NEXT: cvt.u64.u32 %rd14, %r6;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: selp.b16 %rs33, %rs20, %rs8, %p12;
+; SM90-NEXT: selp.b16 %rs34, %rs20, %rs33, %p4;
+; SM90-NEXT: selp.b16 %rs35, %rs19, %rs7, %p11;
+; SM90-NEXT: selp.b16 %rs36, %rs19, %rs35, %p3;
+; SM90-NEXT: mov.b32 %r7, {%rs36, %rs34};
+; SM90-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM90-NEXT: selp.b16 %rs37, %rs18, %rs4, %p10;
+; SM90-NEXT: selp.b16 %rs38, %rs18, %rs37, %p2;
+; SM90-NEXT: selp.b16 %rs39, %rs17, %rs3, %p9;
+; SM90-NEXT: selp.b16 %rs40, %rs17, %rs39, %p1;
+; SM90-NEXT: mov.b32 %r8, {%rs40, %rs38};
+; SM90-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p17, %rd25, 0;
+; SM90-NEXT: @%p17 bra $L__BB149_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @udec_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @udec_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @udec_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [udec_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd16, %rd17}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB153_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd7, %rd17, -1;
+; SM90-NEXT: add.s64 %rd8, %rd16, -1;
+; SM90-NEXT: setp.eq.b64 %p1, %rd17, 0;
+; SM90-NEXT: setp.eq.b64 %p2, %rd16, 0;
+; SM90-NEXT: setp.gt.u64 %p3, %rd17, %rd5;
+; SM90-NEXT: setp.gt.u64 %p4, %rd16, %rd4;
+; SM90-NEXT: selp.b64 %rd9, %rd4, %rd8, %p4;
+; SM90-NEXT: selp.b64 %rd10, %rd4, %rd9, %p2;
+; SM90-NEXT: selp.b64 %rd11, %rd5, %rd7, %p3;
+; SM90-NEXT: selp.b64 %rd12, %rd5, %rd11, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd17};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd13, %rd2, %rd17;
+; SM90-NEXT: xor.b64 %rd14, %rd1, %rd16;
+; SM90-NEXT: or.b64 %rd15, %rd14, %rd13;
+; SM90-NEXT: setp.ne.b64 %p5, %rd15, 0;
+; SM90-NEXT: mov.b64 %rd16, %rd1;
+; SM90-NEXT: mov.b64 %rd17, %rd2;
+; SM90-NEXT: @%p5 bra $L__BB153_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<11>;
+; SM90-NEXT: .reg .b64 %rd<34>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [udec_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd30, %rd31}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB154_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd11, %rd31, -1;
+; SM90-NEXT: add.s64 %rd12, %rd30, -1;
+; SM90-NEXT: setp.eq.b64 %p1, %rd31, 0;
+; SM90-NEXT: setp.eq.b64 %p2, %rd30, 0;
+; SM90-NEXT: setp.gt.u64 %p3, %rd31, %rd2;
+; SM90-NEXT: setp.gt.u64 %p4, %rd30, %rd1;
+; SM90-NEXT: selp.b64 %rd13, %rd1, %rd12, %p4;
+; SM90-NEXT: selp.b64 %rd14, %rd1, %rd13, %p2;
+; SM90-NEXT: selp.b64 %rd15, %rd2, %rd11, %p3;
+; SM90-NEXT: selp.b64 %rd16, %rd2, %rd15, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd30, %rd31};
+; SM90-NEXT: mov.b128 swap, {%rd14, %rd16};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd17, %rd6, %rd31;
+; SM90-NEXT: xor.b64 %rd18, %rd5, %rd30;
+; SM90-NEXT: or.b64 %rd19, %rd18, %rd17;
+; SM90-NEXT: setp.ne.b64 %p5, %rd19, 0;
+; SM90-NEXT: mov.b64 %rd30, %rd5;
+; SM90-NEXT: mov.b64 %rd31, %rd6;
+; SM90-NEXT: @%p5 bra $L__BB154_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd20, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd20, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd20, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd32, %rd33}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB154_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd21, %rd33, -1;
+; SM90-NEXT: add.s64 %rd22, %rd32, -1;
+; SM90-NEXT: setp.eq.b64 %p6, %rd33, 0;
+; SM90-NEXT: setp.eq.b64 %p7, %rd32, 0;
+; SM90-NEXT: setp.gt.u64 %p8, %rd33, %rd4;
+; SM90-NEXT: setp.gt.u64 %p9, %rd32, %rd3;
+; SM90-NEXT: selp.b64 %rd23, %rd3, %rd22, %p9;
+; SM90-NEXT: selp.b64 %rd24, %rd3, %rd23, %p7;
+; SM90-NEXT: selp.b64 %rd25, %rd4, %rd21, %p8;
+; SM90-NEXT: selp.b64 %rd26, %rd4, %rd25, %p6;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd32, %rd33};
+; SM90-NEXT: mov.b128 swap, {%rd24, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd27, %rd8, %rd33;
+; SM90-NEXT: xor.b64 %rd28, %rd7, %rd32;
+; SM90-NEXT: or.b64 %rd29, %rd28, %rd27;
+; SM90-NEXT: setp.ne.b64 %p10, %rd29, 0;
+; SM90-NEXT: mov.b64 %rd32, %rd7;
+; SM90-NEXT: mov.b64 %rd33, %rd8;
+; SM90-NEXT: @%p10 bra $L__BB154_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<21>;
+; SM90-NEXT: .reg .b64 %rd<66>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [udec_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd58, %rd59}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB155_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd59;
+; SM90-NEXT: mov.b64 %rd9, %rd58;
+; SM90-NEXT: add.s64 %rd19, %rd10, -1;
+; SM90-NEXT: add.s64 %rd20, %rd9, -1;
+; SM90-NEXT: setp.eq.b64 %p1, %rd10, 0;
+; SM90-NEXT: setp.eq.b64 %p2, %rd9, 0;
+; SM90-NEXT: setp.gt.u64 %p3, %rd10, %rd6;
+; SM90-NEXT: setp.gt.u64 %p4, %rd9, %rd5;
+; SM90-NEXT: selp.b64 %rd21, %rd5, %rd20, %p4;
+; SM90-NEXT: selp.b64 %rd22, %rd5, %rd21, %p2;
+; SM90-NEXT: selp.b64 %rd23, %rd6, %rd19, %p3;
+; SM90-NEXT: selp.b64 %rd24, %rd6, %rd23, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd22, %rd24};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd58, %rd59}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd59, %rd10;
+; SM90-NEXT: xor.b64 %rd26, %rd58, %rd9;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: setp.ne.b64 %p5, %rd27, 0;
+; SM90-NEXT: @%p5 bra $L__BB155_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd28, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd60, %rd61}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB155_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd61;
+; SM90-NEXT: mov.b64 %rd11, %rd60;
+; SM90-NEXT: add.s64 %rd29, %rd12, -1;
+; SM90-NEXT: add.s64 %rd30, %rd11, -1;
+; SM90-NEXT: setp.eq.b64 %p6, %rd12, 0;
+; SM90-NEXT: setp.eq.b64 %p7, %rd11, 0;
+; SM90-NEXT: setp.gt.u64 %p8, %rd12, %rd8;
+; SM90-NEXT: setp.gt.u64 %p9, %rd11, %rd7;
+; SM90-NEXT: selp.b64 %rd31, %rd7, %rd30, %p9;
+; SM90-NEXT: selp.b64 %rd32, %rd7, %rd31, %p7;
+; SM90-NEXT: selp.b64 %rd33, %rd8, %rd29, %p8;
+; SM90-NEXT: selp.b64 %rd34, %rd8, %rd33, %p6;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd60, %rd61}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd35, %rd61, %rd12;
+; SM90-NEXT: xor.b64 %rd36, %rd60, %rd11;
+; SM90-NEXT: or.b64 %rd37, %rd36, %rd35;
+; SM90-NEXT: setp.ne.b64 %p10, %rd37, 0;
+; SM90-NEXT: @%p10 bra $L__BB155_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd38, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd38, %rd38};
+; SM90-NEXT: mov.b128 swap, {%rd38, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd62, %rd63}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB155_5: // %atomicrmw.start16
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd39, %rd63, -1;
+; SM90-NEXT: add.s64 %rd40, %rd62, -1;
+; SM90-NEXT: setp.eq.b64 %p11, %rd63, 0;
+; SM90-NEXT: setp.eq.b64 %p12, %rd62, 0;
+; SM90-NEXT: setp.gt.u64 %p13, %rd63, %rd2;
+; SM90-NEXT: setp.gt.u64 %p14, %rd62, %rd1;
+; SM90-NEXT: selp.b64 %rd41, %rd1, %rd40, %p14;
+; SM90-NEXT: selp.b64 %rd42, %rd1, %rd41, %p12;
+; SM90-NEXT: selp.b64 %rd43, %rd2, %rd39, %p13;
+; SM90-NEXT: selp.b64 %rd44, %rd2, %rd43, %p11;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd62, %rd63};
+; SM90-NEXT: mov.b128 swap, {%rd42, %rd44};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd45, %rd14, %rd63;
+; SM90-NEXT: xor.b64 %rd46, %rd13, %rd62;
+; SM90-NEXT: or.b64 %rd47, %rd46, %rd45;
+; SM90-NEXT: setp.ne.b64 %p15, %rd47, 0;
+; SM90-NEXT: mov.b64 %rd62, %rd13;
+; SM90-NEXT: mov.b64 %rd63, %rd14;
+; SM90-NEXT: @%p15 bra $L__BB155_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end15
+; SM90-NEXT: mov.b64 %rd48, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd48, %rd48};
+; SM90-NEXT: mov.b128 swap, {%rd48, %rd48};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd64, %rd65}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB155_7: // %atomicrmw.start22
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd49, %rd65, -1;
+; SM90-NEXT: add.s64 %rd50, %rd64, -1;
+; SM90-NEXT: setp.eq.b64 %p16, %rd65, 0;
+; SM90-NEXT: setp.eq.b64 %p17, %rd64, 0;
+; SM90-NEXT: setp.gt.u64 %p18, %rd65, %rd4;
+; SM90-NEXT: setp.gt.u64 %p19, %rd64, %rd3;
+; SM90-NEXT: selp.b64 %rd51, %rd3, %rd50, %p19;
+; SM90-NEXT: selp.b64 %rd52, %rd3, %rd51, %p17;
+; SM90-NEXT: selp.b64 %rd53, %rd4, %rd49, %p18;
+; SM90-NEXT: selp.b64 %rd54, %rd4, %rd53, %p16;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd64, %rd65};
+; SM90-NEXT: mov.b128 swap, {%rd52, %rd54};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd55, %rd16, %rd65;
+; SM90-NEXT: xor.b64 %rd56, %rd15, %rd64;
+; SM90-NEXT: or.b64 %rd57, %rd56, %rd55;
+; SM90-NEXT: setp.ne.b64 %p20, %rd57, 0;
+; SM90-NEXT: mov.b64 %rd64, %rd15;
+; SM90-NEXT: mov.b64 %rd65, %rd16;
+; SM90-NEXT: @%p20 bra $L__BB155_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end21
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b16 %rs<15>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_cond_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: mov.b32 {%rs6, %rs7}, %r4;
+; SM90-NEXT: $L__BB156_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r15, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: and.b16 %rs4, %rs3, 255;
+; SM90-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM90-NEXT: setp.ge.u16 %p1, %rs5, %rs7;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs6;
+; SM90-NEXT: sub.s16 %rs8, %rs5, %rs7;
+; SM90-NEXT: sub.s16 %rs9, %rs3, %rs6;
+; SM90-NEXT: selp.b16 %rs10, %rs9, %rs3, %p2;
+; SM90-NEXT: selp.b16 %rs11, %rs8, %rs5, %p1;
+; SM90-NEXT: shl.b16 %rs12, %rs11, 8;
+; SM90-NEXT: and.b16 %rs13, %rs10, 255;
+; SM90-NEXT: or.b16 %rs14, %rs13, %rs12;
+; SM90-NEXT: cvt.u32.u16 %r10, %rs14;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p3 bra $L__BB156_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<23>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r22, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r9;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r3;
+; SM90-NEXT: $L__BB157_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r22, 0, 0x7770U;
+; SM90-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r22, 0, 0x7771U;
+; SM90-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r8, %r22, 0, 0x7772U;
+; SM90-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r10, %r22, 0, 0x7773U;
+; SM90-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r4;
+; SM90-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM90-NEXT: sub.s16 %rs8, %rs7, %rs3;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r8;
+; SM90-NEXT: sub.s16 %rs10, %rs9, %rs2;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r10;
+; SM90-NEXT: sub.s16 %rs12, %rs11, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r11, %rs12;
+; SM90-NEXT: selp.b32 %r12, %r11, %r10, %p4;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs10;
+; SM90-NEXT: selp.b32 %r14, %r13, %r8, %p3;
+; SM90-NEXT: prmt.b32 %r15, %r14, %r12, 0x3340U;
+; SM90-NEXT: cvt.u32.u16 %r16, %rs8;
+; SM90-NEXT: selp.b32 %r17, %r16, %r6, %p2;
+; SM90-NEXT: cvt.u32.u16 %r18, %rs6;
+; SM90-NEXT: selp.b32 %r19, %r18, %r4, %p1;
+; SM90-NEXT: prmt.b32 %r20, %r19, %r17, 0x3340U;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r15, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r22;
+; SM90-NEXT: mov.b32 %r22, %r1;
+; SM90-NEXT: @%p5 bra $L__BB157_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<33>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r5;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r3;
+; SM90-NEXT: $L__BB158_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM90-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM90-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM90-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM90-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM90-NEXT: setp.ge.u32 %p5, %r12, %r11;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM90-NEXT: setp.ge.u32 %p6, %r14, %r13;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM90-NEXT: setp.ge.u32 %p7, %r16, %r15;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM90-NEXT: setp.ge.u32 %p8, %r18, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r18;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r15;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r16;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r13;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r14;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r11;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r12;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r9;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r10;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r7;
+; SM90-NEXT: cvt.u16.u32 %rs12, %r8;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r6;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r4;
+; SM90-NEXT: sub.s16 %rs17, %rs16, %rs15;
+; SM90-NEXT: sub.s16 %rs18, %rs14, %rs13;
+; SM90-NEXT: sub.s16 %rs19, %rs12, %rs11;
+; SM90-NEXT: sub.s16 %rs20, %rs10, %rs9;
+; SM90-NEXT: sub.s16 %rs21, %rs8, %rs7;
+; SM90-NEXT: sub.s16 %rs22, %rs6, %rs5;
+; SM90-NEXT: sub.s16 %rs23, %rs4, %rs3;
+; SM90-NEXT: sub.s16 %rs24, %rs2, %rs1;
+; SM90-NEXT: selp.b16 %rs25, %rs24, %rs2, %p8;
+; SM90-NEXT: cvt.u32.u16 %r19, %rs25;
+; SM90-NEXT: selp.b16 %rs26, %rs23, %rs4, %p7;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs26;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM90-NEXT: selp.b16 %rs27, %rs22, %rs6, %p6;
+; SM90-NEXT: cvt.u32.u16 %r22, %rs27;
+; SM90-NEXT: selp.b16 %rs28, %rs21, %rs8, %p5;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs28;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM90-NEXT: selp.b16 %rs29, %rs20, %rs10, %p4;
+; SM90-NEXT: cvt.u32.u16 %r26, %rs29;
+; SM90-NEXT: selp.b16 %rs30, %rs19, %rs12, %p3;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs30;
+; SM90-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM90-NEXT: selp.b16 %rs31, %rs18, %rs14, %p2;
+; SM90-NEXT: cvt.u32.u16 %r29, %rs31;
+; SM90-NEXT: selp.b16 %rs32, %rs17, %rs16, %p1;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs32;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB158_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM90-NEXT: $L__BB159_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT: sub.s16 %rs5, %rs3, %rs1;
+; SM90-NEXT: sub.s16 %rs6, %rs4, %rs2;
+; SM90-NEXT: selp.b16 %rs7, %rs6, %rs4, %p2;
+; SM90-NEXT: selp.b16 %rs8, %rs5, %rs3, %p1;
+; SM90-NEXT: mov.b32 %r3, {%rs8, %rs7};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p3, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p3 bra $L__BB159_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b16 %rs<17>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; SM90-NEXT: $L__BB160_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r6;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r5;
+; SM90-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM90-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM90-NEXT: sub.s16 %rs9, %rs3, %rs1;
+; SM90-NEXT: sub.s16 %rs10, %rs4, %rs2;
+; SM90-NEXT: sub.s16 %rs11, %rs7, %rs5;
+; SM90-NEXT: sub.s16 %rs12, %rs8, %rs6;
+; SM90-NEXT: selp.b16 %rs13, %rs12, %rs8, %p4;
+; SM90-NEXT: selp.b16 %rs14, %rs11, %rs7, %p3;
+; SM90-NEXT: mov.b32 %r3, {%rs14, %rs13};
+; SM90-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM90-NEXT: selp.b16 %rs15, %rs10, %rs4, %p2;
+; SM90-NEXT: selp.b16 %rs16, %rs9, %rs3, %p1;
+; SM90-NEXT: mov.b32 %r4, {%rs16, %rs15};
+; SM90-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p5 bra $L__BB160_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<33>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r3;
+; SM90-NEXT: $L__BB161_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r12;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r11;
+; SM90-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM90-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM90-NEXT: mov.b32 {%rs11, %rs12}, %r10;
+; SM90-NEXT: setp.ge.u16 %p5, %rs11, %rs9;
+; SM90-NEXT: setp.ge.u16 %p6, %rs12, %rs10;
+; SM90-NEXT: mov.b32 {%rs13, %rs14}, %r1;
+; SM90-NEXT: mov.b32 {%rs15, %rs16}, %r9;
+; SM90-NEXT: setp.ge.u16 %p7, %rs15, %rs13;
+; SM90-NEXT: setp.ge.u16 %p8, %rs16, %rs14;
+; SM90-NEXT: sub.s16 %rs17, %rs3, %rs1;
+; SM90-NEXT: sub.s16 %rs18, %rs4, %rs2;
+; SM90-NEXT: sub.s16 %rs19, %rs7, %rs5;
+; SM90-NEXT: sub.s16 %rs20, %rs8, %rs6;
+; SM90-NEXT: sub.s16 %rs21, %rs11, %rs9;
+; SM90-NEXT: sub.s16 %rs22, %rs12, %rs10;
+; SM90-NEXT: sub.s16 %rs23, %rs15, %rs13;
+; SM90-NEXT: sub.s16 %rs24, %rs16, %rs14;
+; SM90-NEXT: selp.b16 %rs25, %rs24, %rs16, %p8;
+; SM90-NEXT: selp.b16 %rs26, %rs23, %rs15, %p7;
+; SM90-NEXT: mov.b32 %r5, {%rs26, %rs25};
+; SM90-NEXT: cvt.u64.u32 %rd13, %r5;
+; SM90-NEXT: selp.b16 %rs27, %rs22, %rs12, %p6;
+; SM90-NEXT: selp.b16 %rs28, %rs21, %rs11, %p5;
+; SM90-NEXT: mov.b32 %r6, {%rs28, %rs27};
+; SM90-NEXT: cvt.u64.u32 %rd14, %r6;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: selp.b16 %rs29, %rs20, %rs8, %p4;
+; SM90-NEXT: selp.b16 %rs30, %rs19, %rs7, %p3;
+; SM90-NEXT: mov.b32 %r7, {%rs30, %rs29};
+; SM90-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM90-NEXT: selp.b16 %rs31, %rs18, %rs4, %p2;
+; SM90-NEXT: selp.b16 %rs32, %rs17, %rs3, %p1;
+; SM90-NEXT: mov.b32 %r8, {%rs32, %rs31};
+; SM90-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p9, %rd25, 0;
+; SM90-NEXT: @%p9 bra $L__BB161_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB162_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: setp.ge.u32 %p1, %r8, %r2;
+; SM90-NEXT: setp.ge.u32 %p2, %r7, %r1;
+; SM90-NEXT: sub.s32 %r3, %r8, %r2;
+; SM90-NEXT: sub.s32 %r4, %r7, %r1;
+; SM90-NEXT: selp.b32 %r5, %r4, %r7, %p2;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: selp.b32 %r6, %r3, %r8, %p1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p3 bra $L__BB162_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r16}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd4;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd3;
+; SM90-NEXT: $L__BB163_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r13;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r14;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r15;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r16;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: setp.ge.u32 %p1, %r16, %r4;
+; SM90-NEXT: setp.ge.u32 %p2, %r15, %r3;
+; SM90-NEXT: setp.ge.u32 %p3, %r14, %r2;
+; SM90-NEXT: setp.ge.u32 %p4, %r13, %r1;
+; SM90-NEXT: sub.s32 %r5, %r16, %r4;
+; SM90-NEXT: sub.s32 %r6, %r15, %r3;
+; SM90-NEXT: sub.s32 %r7, %r14, %r2;
+; SM90-NEXT: sub.s32 %r8, %r13, %r1;
+; SM90-NEXT: selp.b32 %r9, %r8, %r13, %p4;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: selp.b32 %r10, %r7, %r14, %p3;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: selp.b32 %r11, %r6, %r15, %p2;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: selp.b32 %r12, %r5, %r16, %p1;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r16}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd22;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd21;
+; SM90-NEXT: setp.ne.b64 %p5, %rd25, 0;
+; SM90-NEXT: @%p5 bra $L__BB163_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<11>;
+; SM90-NEXT: .reg .b32 %r<33>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_cond_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r28}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r27, %rd4;
+; SM90-NEXT: mov.b64 {_, %r26}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r25, %rd3;
+; SM90-NEXT: $L__BB164_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r25;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r26;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r27;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r28;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: setp.ge.u32 %p1, %r28, %r4;
+; SM90-NEXT: setp.ge.u32 %p2, %r27, %r3;
+; SM90-NEXT: setp.ge.u32 %p3, %r26, %r2;
+; SM90-NEXT: setp.ge.u32 %p4, %r25, %r1;
+; SM90-NEXT: sub.s32 %r9, %r28, %r4;
+; SM90-NEXT: sub.s32 %r10, %r27, %r3;
+; SM90-NEXT: sub.s32 %r11, %r26, %r2;
+; SM90-NEXT: sub.s32 %r12, %r25, %r1;
+; SM90-NEXT: selp.b32 %r13, %r12, %r25, %p4;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r13;
+; SM90-NEXT: selp.b32 %r14, %r11, %r26, %p3;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r14;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: selp.b32 %r15, %r10, %r27, %p2;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM90-NEXT: selp.b32 %r16, %r9, %r28, %p1;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r28}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r27, %rd22;
+; SM90-NEXT: mov.b64 {_, %r26}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r25, %rd21;
+; SM90-NEXT: setp.ne.b64 %p5, %rd25, 0;
+; SM90-NEXT: @%p5 bra $L__BB164_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r32}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r31, %rd28;
+; SM90-NEXT: mov.b64 {_, %r30}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r29, %rd27;
+; SM90-NEXT: $L__BB164_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd29, %r29;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r30;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r31;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r32;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: setp.ge.u32 %p6, %r32, %r8;
+; SM90-NEXT: setp.ge.u32 %p7, %r31, %r7;
+; SM90-NEXT: setp.ge.u32 %p8, %r30, %r6;
+; SM90-NEXT: setp.ge.u32 %p9, %r29, %r5;
+; SM90-NEXT: sub.s32 %r17, %r32, %r8;
+; SM90-NEXT: sub.s32 %r18, %r31, %r7;
+; SM90-NEXT: sub.s32 %r19, %r30, %r6;
+; SM90-NEXT: sub.s32 %r20, %r29, %r5;
+; SM90-NEXT: selp.b32 %r21, %r20, %r29, %p9;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r21;
+; SM90-NEXT: selp.b32 %r22, %r19, %r30, %p8;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r22;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: selp.b32 %r23, %r18, %r31, %p7;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r23;
+; SM90-NEXT: selp.b32 %r24, %r17, %r32, %p6;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r24;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd32, %rd36};
+; SM90-NEXT: mov.b128 swap, {%rd40, %rd44};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd36;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd32;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r32}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r31, %rd46;
+; SM90-NEXT: mov.b64 {_, %r30}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r29, %rd45;
+; SM90-NEXT: setp.ne.b64 %p10, %rd49, 0;
+; SM90-NEXT: @%p10 bra $L__BB164_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b64 %rd<16>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd14, %rd15}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB165_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u64 %p1, %rd15, %rd5;
+; SM90-NEXT: setp.ge.u64 %p2, %rd14, %rd4;
+; SM90-NEXT: sub.s64 %rd7, %rd15, %rd5;
+; SM90-NEXT: sub.s64 %rd8, %rd14, %rd4;
+; SM90-NEXT: selp.b64 %rd9, %rd8, %rd14, %p2;
+; SM90-NEXT: selp.b64 %rd10, %rd7, %rd15, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd14, %rd15};
+; SM90-NEXT: mov.b128 swap, {%rd9, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd11, %rd2, %rd15;
+; SM90-NEXT: xor.b64 %rd12, %rd1, %rd14;
+; SM90-NEXT: or.b64 %rd13, %rd12, %rd11;
+; SM90-NEXT: setp.ne.b64 %p3, %rd13, 0;
+; SM90-NEXT: mov.b64 %rd14, %rd1;
+; SM90-NEXT: mov.b64 %rd15, %rd2;
+; SM90-NEXT: @%p3 bra $L__BB165_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<7>;
+; SM90-NEXT: .reg .b64 %rd<30>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [usub_cond_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd26, %rd27}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB166_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u64 %p1, %rd27, %rd2;
+; SM90-NEXT: setp.ge.u64 %p2, %rd26, %rd1;
+; SM90-NEXT: sub.s64 %rd11, %rd27, %rd2;
+; SM90-NEXT: sub.s64 %rd12, %rd26, %rd1;
+; SM90-NEXT: selp.b64 %rd13, %rd12, %rd26, %p2;
+; SM90-NEXT: selp.b64 %rd14, %rd11, %rd27, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd27};
+; SM90-NEXT: mov.b128 swap, {%rd13, %rd14};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd15, %rd6, %rd27;
+; SM90-NEXT: xor.b64 %rd16, %rd5, %rd26;
+; SM90-NEXT: or.b64 %rd17, %rd16, %rd15;
+; SM90-NEXT: setp.ne.b64 %p3, %rd17, 0;
+; SM90-NEXT: mov.b64 %rd26, %rd5;
+; SM90-NEXT: mov.b64 %rd27, %rd6;
+; SM90-NEXT: @%p3 bra $L__BB166_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd28, %rd29}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB166_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u64 %p4, %rd29, %rd4;
+; SM90-NEXT: setp.ge.u64 %p5, %rd28, %rd3;
+; SM90-NEXT: sub.s64 %rd19, %rd29, %rd4;
+; SM90-NEXT: sub.s64 %rd20, %rd28, %rd3;
+; SM90-NEXT: selp.b64 %rd21, %rd20, %rd28, %p5;
+; SM90-NEXT: selp.b64 %rd22, %rd19, %rd29, %p4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd29};
+; SM90-NEXT: mov.b128 swap, {%rd21, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd8, %rd29;
+; SM90-NEXT: xor.b64 %rd24, %rd7, %rd28;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p6, %rd25, 0;
+; SM90-NEXT: mov.b64 %rd28, %rd7;
+; SM90-NEXT: mov.b64 %rd29, %rd8;
+; SM90-NEXT: @%p6 bra $L__BB166_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<13>;
+; SM90-NEXT: .reg .b64 %rd<58>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [usub_cond_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB167_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd51;
+; SM90-NEXT: mov.b64 %rd9, %rd50;
+; SM90-NEXT: setp.ge.u64 %p1, %rd10, %rd6;
+; SM90-NEXT: setp.ge.u64 %p2, %rd9, %rd5;
+; SM90-NEXT: sub.s64 %rd19, %rd10, %rd6;
+; SM90-NEXT: sub.s64 %rd20, %rd9, %rd5;
+; SM90-NEXT: selp.b64 %rd21, %rd20, %rd9, %p2;
+; SM90-NEXT: selp.b64 %rd22, %rd19, %rd10, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd21, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd51, %rd10;
+; SM90-NEXT: xor.b64 %rd24, %rd50, %rd9;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p3, %rd25, 0;
+; SM90-NEXT: @%p3 bra $L__BB167_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB167_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd53;
+; SM90-NEXT: mov.b64 %rd11, %rd52;
+; SM90-NEXT: setp.ge.u64 %p4, %rd12, %rd8;
+; SM90-NEXT: setp.ge.u64 %p5, %rd11, %rd7;
+; SM90-NEXT: sub.s64 %rd27, %rd12, %rd8;
+; SM90-NEXT: sub.s64 %rd28, %rd11, %rd7;
+; SM90-NEXT: selp.b64 %rd29, %rd28, %rd11, %p5;
+; SM90-NEXT: selp.b64 %rd30, %rd27, %rd12, %p4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd29, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd31, %rd53, %rd12;
+; SM90-NEXT: xor.b64 %rd32, %rd52, %rd11;
+; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
+; SM90-NEXT: setp.ne.b64 %p6, %rd33, 0;
+; SM90-NEXT: @%p6 bra $L__BB167_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd34, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd34};
+; SM90-NEXT: mov.b128 swap, {%rd34, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd54, %rd55}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB167_5: // %atomicrmw.start16
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u64 %p7, %rd55, %rd2;
+; SM90-NEXT: setp.ge.u64 %p8, %rd54, %rd1;
+; SM90-NEXT: sub.s64 %rd35, %rd55, %rd2;
+; SM90-NEXT: sub.s64 %rd36, %rd54, %rd1;
+; SM90-NEXT: selp.b64 %rd37, %rd36, %rd54, %p8;
+; SM90-NEXT: selp.b64 %rd38, %rd35, %rd55, %p7;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd54, %rd55};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd14, %rd55;
+; SM90-NEXT: xor.b64 %rd40, %rd13, %rd54;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p9, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd54, %rd13;
+; SM90-NEXT: mov.b64 %rd55, %rd14;
+; SM90-NEXT: @%p9 bra $L__BB167_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end15
+; SM90-NEXT: mov.b64 %rd42, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd42, %rd42};
+; SM90-NEXT: mov.b128 swap, {%rd42, %rd42};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd56, %rd57}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB167_7: // %atomicrmw.start22
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u64 %p10, %rd57, %rd4;
+; SM90-NEXT: setp.ge.u64 %p11, %rd56, %rd3;
+; SM90-NEXT: sub.s64 %rd43, %rd57, %rd4;
+; SM90-NEXT: sub.s64 %rd44, %rd56, %rd3;
+; SM90-NEXT: selp.b64 %rd45, %rd44, %rd56, %p11;
+; SM90-NEXT: selp.b64 %rd46, %rd43, %rd57, %p10;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd56, %rd57};
+; SM90-NEXT: mov.b128 swap, {%rd45, %rd46};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd16, %rd57;
+; SM90-NEXT: xor.b64 %rd48, %rd15, %rd56;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: setp.ne.b64 %p12, %rd49, 0;
+; SM90-NEXT: mov.b64 %rd56, %rd15;
+; SM90-NEXT: mov.b64 %rd57, %rd16;
+; SM90-NEXT: @%p12 bra $L__BB167_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end21
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_sat_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM90-NEXT: $L__BB168_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, 16711935;
+; SM90-NEXT: max.u16x2 %r12, %r11, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: sub.s16 %rs9, %rs5, %rs7;
+; SM90-NEXT: sub.s16 %rs10, %rs6, %rs8;
+; SM90-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM90-NEXT: or.b16 %rs12, %rs9, %rs11;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs12;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB168_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<17>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r13;
+; SM90-NEXT: $L__BB169_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM90-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r7;
+; SM90-NEXT: max.u16 %rs7, %rs6, %rs5;
+; SM90-NEXT: sub.s16 %rs8, %rs7, %rs5;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs8;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r11;
+; SM90-NEXT: max.u16 %rs11, %rs10, %rs9;
+; SM90-NEXT: sub.s16 %rs12, %rs11, %rs9;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r14;
+; SM90-NEXT: max.u16 %rs15, %rs14, %rs13;
+; SM90-NEXT: sub.s16 %rs16, %rs15, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs16;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB169_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<33>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB170_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM90-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r7;
+; SM90-NEXT: max.u16 %rs7, %rs6, %rs5;
+; SM90-NEXT: sub.s16 %rs8, %rs7, %rs5;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs8;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r11;
+; SM90-NEXT: max.u16 %rs11, %rs10, %rs9;
+; SM90-NEXT: sub.s16 %rs12, %rs11, %rs9;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r14;
+; SM90-NEXT: max.u16 %rs15, %rs14, %rs13;
+; SM90-NEXT: sub.s16 %rs16, %rs15, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs16;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs18, %r19;
+; SM90-NEXT: max.u16 %rs19, %rs18, %rs17;
+; SM90-NEXT: sub.s16 %rs20, %rs19, %rs17;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM90-NEXT: cvt.u16.u32 %rs21, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r22;
+; SM90-NEXT: max.u16 %rs23, %rs22, %rs21;
+; SM90-NEXT: sub.s16 %rs24, %rs23, %rs21;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs24;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs25, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs26, %r26;
+; SM90-NEXT: max.u16 %rs27, %rs26, %rs25;
+; SM90-NEXT: sub.s16 %rs28, %rs27, %rs25;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs28;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs29, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs30, %r29;
+; SM90-NEXT: max.u16 %rs31, %rs30, %rs29;
+; SM90-NEXT: sub.s16 %rs32, %rs31, %rs29;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs32;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB170_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<7>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r2;
+; SM90-NEXT: $L__BB171_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u16x2 %r3, %r5, %r2;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; SM90-NEXT: sub.s16 %rs5, %rs2, %rs4;
+; SM90-NEXT: sub.s16 %rs6, %rs1, %rs3;
+; SM90-NEXT: mov.b32 %r4, {%rs6, %rs5};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB171_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r1;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM90-NEXT: $L__BB172_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u16x2 %r3, %r7, %r1;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; SM90-NEXT: sub.s16 %rs5, %rs2, %rs4;
+; SM90-NEXT: sub.s16 %rs6, %rs1, %rs3;
+; SM90-NEXT: mov.b32 %r4, {%rs6, %rs5};
+; SM90-NEXT: cvt.u64.u32 %rd3, %r4;
+; SM90-NEXT: max.u16x2 %r5, %r8, %r2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r5;
+; SM90-NEXT: sub.s16 %rs11, %rs8, %rs10;
+; SM90-NEXT: sub.s16 %rs12, %rs7, %rs9;
+; SM90-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB172_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r16}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd4;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd3;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r1;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM90-NEXT: $L__BB173_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u16x2 %r5, %r13, %r1;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r5;
+; SM90-NEXT: sub.s16 %rs5, %rs2, %rs4;
+; SM90-NEXT: sub.s16 %rs6, %rs1, %rs3;
+; SM90-NEXT: mov.b32 %r6, {%rs6, %rs5};
+; SM90-NEXT: cvt.u64.u32 %rd5, %r6;
+; SM90-NEXT: max.u16x2 %r7, %r14, %r2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r7;
+; SM90-NEXT: sub.s16 %rs11, %rs8, %rs10;
+; SM90-NEXT: sub.s16 %rs12, %rs7, %rs9;
+; SM90-NEXT: mov.b32 %r8, {%rs12, %rs11};
+; SM90-NEXT: cvt.u64.u32 %rd6, %r8;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.u16x2 %r9, %r15, %r3;
+; SM90-NEXT: mov.b32 {%rs13, %rs14}, %r9;
+; SM90-NEXT: mov.b32 {%rs15, %rs16}, %r3;
+; SM90-NEXT: sub.s16 %rs17, %rs14, %rs16;
+; SM90-NEXT: sub.s16 %rs18, %rs13, %rs15;
+; SM90-NEXT: mov.b32 %r10, {%rs18, %rs17};
+; SM90-NEXT: cvt.u64.u32 %rd9, %r10;
+; SM90-NEXT: max.u16x2 %r11, %r16, %r4;
+; SM90-NEXT: mov.b32 {%rs19, %rs20}, %r11;
+; SM90-NEXT: mov.b32 {%rs21, %rs22}, %r4;
+; SM90-NEXT: sub.s16 %rs23, %rs20, %rs22;
+; SM90-NEXT: sub.s16 %rs24, %rs19, %rs21;
+; SM90-NEXT: mov.b32 %r12, {%rs24, %rs23};
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r13;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r14;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r16}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd22;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB173_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB174_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u32 %r3, %r7, %r1;
+; SM90-NEXT: sub.s32 %r4, %r3, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r4;
+; SM90-NEXT: max.u32 %r5, %r8, %r2;
+; SM90-NEXT: sub.s32 %r6, %r5, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB174_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r16}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd4;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd3;
+; SM90-NEXT: $L__BB175_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u32 %r5, %r13, %r1;
+; SM90-NEXT: sub.s32 %r6, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r6;
+; SM90-NEXT: max.u32 %r7, %r14, %r2;
+; SM90-NEXT: sub.s32 %r8, %r7, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r8;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.u32 %r9, %r15, %r3;
+; SM90-NEXT: sub.s32 %r10, %r9, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r10;
+; SM90-NEXT: max.u32 %r11, %r16, %r4;
+; SM90-NEXT: sub.s32 %r12, %r11, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r13;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r14;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r16}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd22;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB175_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<33>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_sat_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r28}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r27, %rd4;
+; SM90-NEXT: mov.b64 {_, %r26}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r25, %rd3;
+; SM90-NEXT: $L__BB176_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u32 %r9, %r25, %r1;
+; SM90-NEXT: sub.s32 %r10, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r10;
+; SM90-NEXT: max.u32 %r11, %r26, %r2;
+; SM90-NEXT: sub.s32 %r12, %r11, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r12;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.u32 %r13, %r27, %r3;
+; SM90-NEXT: sub.s32 %r14, %r13, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r14;
+; SM90-NEXT: max.u32 %r15, %r28, %r4;
+; SM90-NEXT: sub.s32 %r16, %r15, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r16;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r26;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r28}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r27, %rd22;
+; SM90-NEXT: mov.b64 {_, %r26}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r25, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB176_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r32}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r31, %rd28;
+; SM90-NEXT: mov.b64 {_, %r30}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r29, %rd27;
+; SM90-NEXT: $L__BB176_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u32 %r17, %r29, %r5;
+; SM90-NEXT: sub.s32 %r18, %r17, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd29, %r18;
+; SM90-NEXT: max.u32 %r19, %r30, %r6;
+; SM90-NEXT: sub.s32 %r20, %r19, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r20;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: max.u32 %r21, %r31, %r7;
+; SM90-NEXT: sub.s32 %r22, %r21, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r22;
+; SM90-NEXT: max.u32 %r23, %r32, %r8;
+; SM90-NEXT: sub.s32 %r24, %r23, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r24;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r29;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r30;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r31;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r32;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd44};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd44;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd40;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r32}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r31, %rd46;
+; SM90-NEXT: mov.b64 {_, %r30}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r29, %rd45;
+; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
+; SM90-NEXT: @%p2 bra $L__BB176_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<16>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd14, %rd15}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB177_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u64 %rd7, %rd14, %rd4;
+; SM90-NEXT: sub.s64 %rd8, %rd7, %rd4;
+; SM90-NEXT: max.u64 %rd9, %rd15, %rd5;
+; SM90-NEXT: sub.s64 %rd10, %rd9, %rd5;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd14, %rd15};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd11, %rd2, %rd15;
+; SM90-NEXT: xor.b64 %rd12, %rd1, %rd14;
+; SM90-NEXT: or.b64 %rd13, %rd12, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd13, 0;
+; SM90-NEXT: mov.b64 %rd14, %rd1;
+; SM90-NEXT: mov.b64 %rd15, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB177_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<30>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [usub_sat_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd26, %rd27}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB178_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u64 %rd11, %rd26, %rd1;
+; SM90-NEXT: sub.s64 %rd12, %rd11, %rd1;
+; SM90-NEXT: max.u64 %rd13, %rd27, %rd2;
+; SM90-NEXT: sub.s64 %rd14, %rd13, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd27};
+; SM90-NEXT: mov.b128 swap, {%rd12, %rd14};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd15, %rd6, %rd27;
+; SM90-NEXT: xor.b64 %rd16, %rd5, %rd26;
+; SM90-NEXT: or.b64 %rd17, %rd16, %rd15;
+; SM90-NEXT: setp.ne.b64 %p1, %rd17, 0;
+; SM90-NEXT: mov.b64 %rd26, %rd5;
+; SM90-NEXT: mov.b64 %rd27, %rd6;
+; SM90-NEXT: @%p1 bra $L__BB178_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd28, %rd29}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB178_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u64 %rd19, %rd28, %rd3;
+; SM90-NEXT: sub.s64 %rd20, %rd19, %rd3;
+; SM90-NEXT: max.u64 %rd21, %rd29, %rd4;
+; SM90-NEXT: sub.s64 %rd22, %rd21, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd29};
+; SM90-NEXT: mov.b128 swap, {%rd20, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd8, %rd29;
+; SM90-NEXT: xor.b64 %rd24, %rd7, %rd28;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p2, %rd25, 0;
+; SM90-NEXT: mov.b64 %rd28, %rd7;
+; SM90-NEXT: mov.b64 %rd29, %rd8;
+; SM90-NEXT: @%p2 bra $L__BB178_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<5>;
+; SM90-NEXT: .reg .b64 %rd<58>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [usub_sat_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB179_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd51;
+; SM90-NEXT: mov.b64 %rd9, %rd50;
+; SM90-NEXT: max.u64 %rd19, %rd9, %rd5;
+; SM90-NEXT: sub.s64 %rd20, %rd19, %rd5;
+; SM90-NEXT: max.u64 %rd21, %rd10, %rd6;
+; SM90-NEXT: sub.s64 %rd22, %rd21, %rd6;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd20, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd51, %rd10;
+; SM90-NEXT: xor.b64 %rd24, %rd50, %rd9;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB179_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB179_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd53;
+; SM90-NEXT: mov.b64 %rd11, %rd52;
+; SM90-NEXT: max.u64 %rd27, %rd11, %rd7;
+; SM90-NEXT: sub.s64 %rd28, %rd27, %rd7;
+; SM90-NEXT: max.u64 %rd29, %rd12, %rd8;
+; SM90-NEXT: sub.s64 %rd30, %rd29, %rd8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd31, %rd53, %rd12;
+; SM90-NEXT: xor.b64 %rd32, %rd52, %rd11;
+; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
+; SM90-NEXT: setp.ne.b64 %p2, %rd33, 0;
+; SM90-NEXT: @%p2 bra $L__BB179_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd34, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd34};
+; SM90-NEXT: mov.b128 swap, {%rd34, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd54, %rd55}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB179_5: // %atomicrmw.start16
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u64 %rd35, %rd54, %rd1;
+; SM90-NEXT: sub.s64 %rd36, %rd35, %rd1;
+; SM90-NEXT: max.u64 %rd37, %rd55, %rd2;
+; SM90-NEXT: sub.s64 %rd38, %rd37, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd54, %rd55};
+; SM90-NEXT: mov.b128 swap, {%rd36, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd14, %rd55;
+; SM90-NEXT: xor.b64 %rd40, %rd13, %rd54;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p3, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd54, %rd13;
+; SM90-NEXT: mov.b64 %rd55, %rd14;
+; SM90-NEXT: @%p3 bra $L__BB179_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end15
+; SM90-NEXT: mov.b64 %rd42, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd42, %rd42};
+; SM90-NEXT: mov.b128 swap, {%rd42, %rd42};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd56, %rd57}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB179_7: // %atomicrmw.start22
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u64 %rd43, %rd56, %rd3;
+; SM90-NEXT: sub.s64 %rd44, %rd43, %rd3;
+; SM90-NEXT: max.u64 %rd45, %rd57, %rd4;
+; SM90-NEXT: sub.s64 %rd46, %rd45, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd56, %rd57};
+; SM90-NEXT: mov.b128 swap, {%rd44, %rd46};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd16, %rd57;
+; SM90-NEXT: xor.b64 %rd48, %rd15, %rd56;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: setp.ne.b64 %p4, %rd49, 0;
+; SM90-NEXT: mov.b64 %rd56, %rd15;
+; SM90-NEXT: mov.b64 %rd57, %rd16;
+; SM90-NEXT: @%p4 bra $L__BB179_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end21
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM90-LABEL: fadd_acq_rel_v2f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM90-LABEL: fadd_acq_rel_v4f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM90-LABEL: fadd_acq_rel_v8f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM90-LABEL: fsub_acq_rel_v2f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB183_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: sub.rn.f32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB183_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM90-LABEL: fsub_acq_rel_v4f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB184_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: sub.rn.f32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: sub.rn.f32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: sub.rn.f32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB184_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM90-LABEL: fsub_acq_rel_v8f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB185_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: sub.rn.f32 %r10, %r18, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: sub.rn.f32 %r11, %r19, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: sub.rn.f32 %r12, %r20, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r18;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd22;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB185_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
+; SM90-NEXT: $L__BB185_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f32 %r13, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
+; SM90-NEXT: sub.rn.f32 %r14, %r22, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r14;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: sub.rn.f32 %r15, %r23, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM90-NEXT: sub.rn.f32 %r16, %r24, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r22;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r24;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd44};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd44;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd40;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd46;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
+; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
+; SM90-NEXT: @%p2 bra $L__BB185_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM90-LABEL: fmin_acq_rel_v2f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB186_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: min.f32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB186_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM90-LABEL: fmin_acq_rel_v4f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB187_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: min.f32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.f32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: min.f32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB187_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM90-LABEL: fmin_acq_rel_v8f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmin_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB188_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: min.f32 %r10, %r18, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.f32 %r11, %r19, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: min.f32 %r12, %r20, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r18;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd22;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB188_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
+; SM90-NEXT: $L__BB188_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f32 %r13, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
+; SM90-NEXT: min.f32 %r14, %r22, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r14;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: min.f32 %r15, %r23, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM90-NEXT: min.f32 %r16, %r24, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r22;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r24;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd44};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd44;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd40;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd46;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
+; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
+; SM90-NEXT: @%p2 bra $L__BB188_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM90-LABEL: fmax_acq_rel_v2f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB189_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: max.f32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB189_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM90-LABEL: fmax_acq_rel_v4f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB190_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: max.f32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.f32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: max.f32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB190_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM90-LABEL: fmax_acq_rel_v8f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmax_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB191_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: max.f32 %r10, %r18, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.f32 %r11, %r19, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: max.f32 %r12, %r20, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r18;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd22;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB191_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
+; SM90-NEXT: $L__BB191_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f32 %r13, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
+; SM90-NEXT: max.f32 %r14, %r22, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r14;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: max.f32 %r15, %r23, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM90-NEXT: max.f32 %r16, %r24, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r22;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r24;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd44};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd44;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd40;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd46;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
+; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
+; SM90-NEXT: @%p2 bra $L__BB191_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM90-LABEL: fminimum_acq_rel_v2f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB192_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: min.NaN.f32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB192_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM90-LABEL: fminimum_acq_rel_v4f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB193_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: min.NaN.f32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.NaN.f32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: min.NaN.f32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB193_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM90-LABEL: fminimum_acq_rel_v8f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB194_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: min.NaN.f32 %r10, %r18, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.NaN.f32 %r11, %r19, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: min.NaN.f32 %r12, %r20, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r18;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd22;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB194_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
+; SM90-NEXT: $L__BB194_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f32 %r13, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
+; SM90-NEXT: min.NaN.f32 %r14, %r22, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r14;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: min.NaN.f32 %r15, %r23, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM90-NEXT: min.NaN.f32 %r16, %r24, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r22;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r24;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd44};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd44;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd40;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd46;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
+; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
+; SM90-NEXT: @%p2 bra $L__BB194_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v2f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB195_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: max.NaN.f32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB195_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v4f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB196_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: max.NaN.f32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.NaN.f32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: max.NaN.f32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB196_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v8f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB197_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: max.NaN.f32 %r10, %r18, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.NaN.f32 %r11, %r19, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: max.NaN.f32 %r12, %r20, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r18;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd22;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB197_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
+; SM90-NEXT: $L__BB197_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f32 %r13, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
+; SM90-NEXT: max.NaN.f32 %r14, %r22, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r14;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: max.NaN.f32 %r15, %r23, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM90-NEXT: max.NaN.f32 %r16, %r24, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r22;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r24;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd44};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd44;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd40;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd46;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
+; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
+; SM90-NEXT: @%p2 bra $L__BB197_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <2 x double> @fadd_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM90-LABEL: fadd_acq_rel_v2f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fadd_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM90-LABEL: fadd_acq_rel_v4f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fadd_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM90-LABEL: fadd_acq_rel_v8f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [fadd_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [fadd_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM90-LABEL: fsub_acq_rel_v2f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<14>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fsub_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [fsub_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd12, %rd13}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB201_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f64 %rd7, %rd12, %rd4;
+; SM90-NEXT: sub.rn.f64 %rd8, %rd13, %rd5;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd12, %rd13};
+; SM90-NEXT: mov.b128 swap, {%rd7, %rd8};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd9, %rd2, %rd13;
+; SM90-NEXT: xor.b64 %rd10, %rd1, %rd12;
+; SM90-NEXT: or.b64 %rd11, %rd10, %rd9;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, 0;
+; SM90-NEXT: mov.b64 %rd12, %rd1;
+; SM90-NEXT: mov.b64 %rd13, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB201_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM90-LABEL: fsub_acq_rel_v4f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [fsub_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd22, %rd23}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB202_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f64 %rd11, %rd22, %rd1;
+; SM90-NEXT: sub.rn.f64 %rd12, %rd23, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd22, %rd23};
+; SM90-NEXT: mov.b128 swap, {%rd11, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd13, %rd6, %rd23;
+; SM90-NEXT: xor.b64 %rd14, %rd5, %rd22;
+; SM90-NEXT: or.b64 %rd15, %rd14, %rd13;
+; SM90-NEXT: setp.ne.b64 %p1, %rd15, 0;
+; SM90-NEXT: mov.b64 %rd22, %rd5;
+; SM90-NEXT: mov.b64 %rd23, %rd6;
+; SM90-NEXT: @%p1 bra $L__BB202_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd16, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd16};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd16};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd24, %rd25}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB202_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f64 %rd17, %rd24, %rd3;
+; SM90-NEXT: sub.rn.f64 %rd18, %rd25, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd25};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd19, %rd8, %rd25;
+; SM90-NEXT: xor.b64 %rd20, %rd7, %rd24;
+; SM90-NEXT: or.b64 %rd21, %rd20, %rd19;
+; SM90-NEXT: setp.ne.b64 %p2, %rd21, 0;
+; SM90-NEXT: mov.b64 %rd24, %rd7;
+; SM90-NEXT: mov.b64 %rd25, %rd8;
+; SM90-NEXT: @%p2 bra $L__BB202_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM90-LABEL: fsub_acq_rel_v8f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<5>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [fsub_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fsub_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB203_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd43;
+; SM90-NEXT: mov.b64 %rd9, %rd42;
+; SM90-NEXT: sub.rn.f64 %rd19, %rd9, %rd5;
+; SM90-NEXT: sub.rn.f64 %rd20, %rd10, %rd6;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd19, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd21, %rd43, %rd10;
+; SM90-NEXT: xor.b64 %rd22, %rd42, %rd9;
+; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd23, 0;
+; SM90-NEXT: @%p1 bra $L__BB203_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd24, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd24};
+; SM90-NEXT: mov.b128 swap, {%rd24, %rd24};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB203_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd45;
+; SM90-NEXT: mov.b64 %rd11, %rd44;
+; SM90-NEXT: sub.rn.f64 %rd25, %rd11, %rd7;
+; SM90-NEXT: sub.rn.f64 %rd26, %rd12, %rd8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd25, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd27, %rd45, %rd12;
+; SM90-NEXT: xor.b64 %rd28, %rd44, %rd11;
+; SM90-NEXT: or.b64 %rd29, %rd28, %rd27;
+; SM90-NEXT: setp.ne.b64 %p2, %rd29, 0;
+; SM90-NEXT: @%p2 bra $L__BB203_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd30, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd30, %rd30};
+; SM90-NEXT: mov.b128 swap, {%rd30, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd46, %rd47}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB203_5: // %atomicrmw.start16
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f64 %rd31, %rd46, %rd1;
+; SM90-NEXT: sub.rn.f64 %rd32, %rd47, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd46, %rd47};
+; SM90-NEXT: mov.b128 swap, {%rd31, %rd32};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd33, %rd14, %rd47;
+; SM90-NEXT: xor.b64 %rd34, %rd13, %rd46;
+; SM90-NEXT: or.b64 %rd35, %rd34, %rd33;
+; SM90-NEXT: setp.ne.b64 %p3, %rd35, 0;
+; SM90-NEXT: mov.b64 %rd46, %rd13;
+; SM90-NEXT: mov.b64 %rd47, %rd14;
+; SM90-NEXT: @%p3 bra $L__BB203_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end15
+; SM90-NEXT: mov.b64 %rd36, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd36, %rd36};
+; SM90-NEXT: mov.b128 swap, {%rd36, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd48, %rd49}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB203_7: // %atomicrmw.start22
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f64 %rd37, %rd48, %rd3;
+; SM90-NEXT: sub.rn.f64 %rd38, %rd49, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd48, %rd49};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd16, %rd49;
+; SM90-NEXT: xor.b64 %rd40, %rd15, %rd48;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p4, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd48, %rd15;
+; SM90-NEXT: mov.b64 %rd49, %rd16;
+; SM90-NEXT: @%p4 bra $L__BB203_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end21
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM90-LABEL: fmin_acq_rel_v2f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<14>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fmin_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd12, %rd13}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB204_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f64 %rd7, %rd12, %rd4;
+; SM90-NEXT: min.f64 %rd8, %rd13, %rd5;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd12, %rd13};
+; SM90-NEXT: mov.b128 swap, {%rd7, %rd8};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd9, %rd2, %rd13;
+; SM90-NEXT: xor.b64 %rd10, %rd1, %rd12;
+; SM90-NEXT: or.b64 %rd11, %rd10, %rd9;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, 0;
+; SM90-NEXT: mov.b64 %rd12, %rd1;
+; SM90-NEXT: mov.b64 %rd13, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB204_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM90-LABEL: fmin_acq_rel_v4f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [fmin_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd22, %rd23}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB205_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f64 %rd11, %rd22, %rd1;
+; SM90-NEXT: min.f64 %rd12, %rd23, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd22, %rd23};
+; SM90-NEXT: mov.b128 swap, {%rd11, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd13, %rd6, %rd23;
+; SM90-NEXT: xor.b64 %rd14, %rd5, %rd22;
+; SM90-NEXT: or.b64 %rd15, %rd14, %rd13;
+; SM90-NEXT: setp.ne.b64 %p1, %rd15, 0;
+; SM90-NEXT: mov.b64 %rd22, %rd5;
+; SM90-NEXT: mov.b64 %rd23, %rd6;
+; SM90-NEXT: @%p1 bra $L__BB205_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd16, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd16};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd16};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd24, %rd25}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB205_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f64 %rd17, %rd24, %rd3;
+; SM90-NEXT: min.f64 %rd18, %rd25, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd25};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd19, %rd8, %rd25;
+; SM90-NEXT: xor.b64 %rd20, %rd7, %rd24;
+; SM90-NEXT: or.b64 %rd21, %rd20, %rd19;
+; SM90-NEXT: setp.ne.b64 %p2, %rd21, 0;
+; SM90-NEXT: mov.b64 %rd24, %rd7;
+; SM90-NEXT: mov.b64 %rd25, %rd8;
+; SM90-NEXT: @%p2 bra $L__BB205_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM90-LABEL: fmin_acq_rel_v8f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<5>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [fmin_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmin_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB206_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd43;
+; SM90-NEXT: mov.b64 %rd9, %rd42;
+; SM90-NEXT: min.f64 %rd19, %rd9, %rd5;
+; SM90-NEXT: min.f64 %rd20, %rd10, %rd6;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd19, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd21, %rd43, %rd10;
+; SM90-NEXT: xor.b64 %rd22, %rd42, %rd9;
+; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd23, 0;
+; SM90-NEXT: @%p1 bra $L__BB206_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd24, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd24};
+; SM90-NEXT: mov.b128 swap, {%rd24, %rd24};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB206_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd45;
+; SM90-NEXT: mov.b64 %rd11, %rd44;
+; SM90-NEXT: min.f64 %rd25, %rd11, %rd7;
+; SM90-NEXT: min.f64 %rd26, %rd12, %rd8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd25, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd27, %rd45, %rd12;
+; SM90-NEXT: xor.b64 %rd28, %rd44, %rd11;
+; SM90-NEXT: or.b64 %rd29, %rd28, %rd27;
+; SM90-NEXT: setp.ne.b64 %p2, %rd29, 0;
+; SM90-NEXT: @%p2 bra $L__BB206_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd30, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd30, %rd30};
+; SM90-NEXT: mov.b128 swap, {%rd30, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd46, %rd47}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB206_5: // %atomicrmw.start15
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f64 %rd31, %rd46, %rd1;
+; SM90-NEXT: min.f64 %rd32, %rd47, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd46, %rd47};
+; SM90-NEXT: mov.b128 swap, {%rd31, %rd32};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd33, %rd14, %rd47;
+; SM90-NEXT: xor.b64 %rd34, %rd13, %rd46;
+; SM90-NEXT: or.b64 %rd35, %rd34, %rd33;
+; SM90-NEXT: setp.ne.b64 %p3, %rd35, 0;
+; SM90-NEXT: mov.b64 %rd46, %rd13;
+; SM90-NEXT: mov.b64 %rd47, %rd14;
+; SM90-NEXT: @%p3 bra $L__BB206_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end14
+; SM90-NEXT: mov.b64 %rd36, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd36, %rd36};
+; SM90-NEXT: mov.b128 swap, {%rd36, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd48, %rd49}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB206_7: // %atomicrmw.start20
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f64 %rd37, %rd48, %rd3;
+; SM90-NEXT: min.f64 %rd38, %rd49, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd48, %rd49};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd16, %rd49;
+; SM90-NEXT: xor.b64 %rd40, %rd15, %rd48;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p4, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd48, %rd15;
+; SM90-NEXT: mov.b64 %rd49, %rd16;
+; SM90-NEXT: @%p4 bra $L__BB206_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end19
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM90-LABEL: fmax_acq_rel_v2f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<14>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fmax_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd12, %rd13}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB207_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f64 %rd7, %rd12, %rd4;
+; SM90-NEXT: max.f64 %rd8, %rd13, %rd5;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd12, %rd13};
+; SM90-NEXT: mov.b128 swap, {%rd7, %rd8};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd9, %rd2, %rd13;
+; SM90-NEXT: xor.b64 %rd10, %rd1, %rd12;
+; SM90-NEXT: or.b64 %rd11, %rd10, %rd9;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, 0;
+; SM90-NEXT: mov.b64 %rd12, %rd1;
+; SM90-NEXT: mov.b64 %rd13, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB207_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM90-LABEL: fmax_acq_rel_v4f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [fmax_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd22, %rd23}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB208_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f64 %rd11, %rd22, %rd1;
+; SM90-NEXT: max.f64 %rd12, %rd23, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd22, %rd23};
+; SM90-NEXT: mov.b128 swap, {%rd11, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd13, %rd6, %rd23;
+; SM90-NEXT: xor.b64 %rd14, %rd5, %rd22;
+; SM90-NEXT: or.b64 %rd15, %rd14, %rd13;
+; SM90-NEXT: setp.ne.b64 %p1, %rd15, 0;
+; SM90-NEXT: mov.b64 %rd22, %rd5;
+; SM90-NEXT: mov.b64 %rd23, %rd6;
+; SM90-NEXT: @%p1 bra $L__BB208_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd16, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd16};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd16};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd24, %rd25}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB208_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f64 %rd17, %rd24, %rd3;
+; SM90-NEXT: max.f64 %rd18, %rd25, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd25};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd19, %rd8, %rd25;
+; SM90-NEXT: xor.b64 %rd20, %rd7, %rd24;
+; SM90-NEXT: or.b64 %rd21, %rd20, %rd19;
+; SM90-NEXT: setp.ne.b64 %p2, %rd21, 0;
+; SM90-NEXT: mov.b64 %rd24, %rd7;
+; SM90-NEXT: mov.b64 %rd25, %rd8;
+; SM90-NEXT: @%p2 bra $L__BB208_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM90-LABEL: fmax_acq_rel_v8f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<5>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [fmax_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmax_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB209_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd43;
+; SM90-NEXT: mov.b64 %rd9, %rd42;
+; SM90-NEXT: max.f64 %rd19, %rd9, %rd5;
+; SM90-NEXT: max.f64 %rd20, %rd10, %rd6;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd19, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd21, %rd43, %rd10;
+; SM90-NEXT: xor.b64 %rd22, %rd42, %rd9;
+; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd23, 0;
+; SM90-NEXT: @%p1 bra $L__BB209_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd24, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd24};
+; SM90-NEXT: mov.b128 swap, {%rd24, %rd24};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB209_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd45;
+; SM90-NEXT: mov.b64 %rd11, %rd44;
+; SM90-NEXT: max.f64 %rd25, %rd11, %rd7;
+; SM90-NEXT: max.f64 %rd26, %rd12, %rd8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd25, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd27, %rd45, %rd12;
+; SM90-NEXT: xor.b64 %rd28, %rd44, %rd11;
+; SM90-NEXT: or.b64 %rd29, %rd28, %rd27;
+; SM90-NEXT: setp.ne.b64 %p2, %rd29, 0;
+; SM90-NEXT: @%p2 bra $L__BB209_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd30, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd30, %rd30};
+; SM90-NEXT: mov.b128 swap, {%rd30, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd46, %rd47}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB209_5: // %atomicrmw.start15
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f64 %rd31, %rd46, %rd1;
+; SM90-NEXT: max.f64 %rd32, %rd47, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd46, %rd47};
+; SM90-NEXT: mov.b128 swap, {%rd31, %rd32};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd33, %rd14, %rd47;
+; SM90-NEXT: xor.b64 %rd34, %rd13, %rd46;
+; SM90-NEXT: or.b64 %rd35, %rd34, %rd33;
+; SM90-NEXT: setp.ne.b64 %p3, %rd35, 0;
+; SM90-NEXT: mov.b64 %rd46, %rd13;
+; SM90-NEXT: mov.b64 %rd47, %rd14;
+; SM90-NEXT: @%p3 bra $L__BB209_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end14
+; SM90-NEXT: mov.b64 %rd36, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd36, %rd36};
+; SM90-NEXT: mov.b128 swap, {%rd36, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd48, %rd49}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB209_7: // %atomicrmw.start20
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f64 %rd37, %rd48, %rd3;
+; SM90-NEXT: max.f64 %rd38, %rd49, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd48, %rd49};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd16, %rd49;
+; SM90-NEXT: xor.b64 %rd40, %rd15, %rd48;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p4, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd48, %rd15;
+; SM90-NEXT: mov.b64 %rd49, %rd16;
+; SM90-NEXT: @%p4 bra $L__BB209_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end19
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM90-LABEL: fminimum_acq_rel_v2f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b64 %rd<22>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fminimum_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd20, %rd21}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p3, %rd4, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p7, %rd5, -9223372036854775808;
+; SM90-NEXT: $L__BB210_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p1, %rd20, %rd4;
+; SM90-NEXT: min.f64 %rd7, %rd20, %rd4;
+; SM90-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd20, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd9, %rd20, %rd8, %p2;
+; SM90-NEXT: selp.f64 %rd10, %rd4, %rd9, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd8, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd11, %rd10, %rd8, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd21, %rd5;
+; SM90-NEXT: min.f64 %rd12, %rd21, %rd5;
+; SM90-NEXT: selp.f64 %rd13, 0d7FF8000000000000, %rd12, %p5;
+; SM90-NEXT: setp.eq.b64 %p6, %rd21, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd14, %rd21, %rd13, %p6;
+; SM90-NEXT: selp.f64 %rd15, %rd5, %rd14, %p7;
+; SM90-NEXT: setp.eq.f64 %p8, %rd13, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd16, %rd15, %rd13, %p8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd20, %rd21};
+; SM90-NEXT: mov.b128 swap, {%rd11, %rd16};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd17, %rd2, %rd21;
+; SM90-NEXT: xor.b64 %rd18, %rd1, %rd20;
+; SM90-NEXT: or.b64 %rd19, %rd18, %rd17;
+; SM90-NEXT: setp.ne.b64 %p9, %rd19, 0;
+; SM90-NEXT: mov.b64 %rd20, %rd1;
+; SM90-NEXT: mov.b64 %rd21, %rd2;
+; SM90-NEXT: @%p9 bra $L__BB210_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM90-LABEL: fminimum_acq_rel_v4f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<19>;
+; SM90-NEXT: .reg .b64 %rd<42>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd38, %rd39}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p7, %rd2, -9223372036854775808;
+; SM90-NEXT: $L__BB211_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p1, %rd38, %rd1;
+; SM90-NEXT: min.f64 %rd11, %rd38, %rd1;
+; SM90-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd38, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd13, %rd38, %rd12, %p2;
+; SM90-NEXT: selp.f64 %rd14, %rd1, %rd13, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd12, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd15, %rd14, %rd12, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd39, %rd2;
+; SM90-NEXT: min.f64 %rd16, %rd39, %rd2;
+; SM90-NEXT: selp.f64 %rd17, 0d7FF8000000000000, %rd16, %p5;
+; SM90-NEXT: setp.eq.b64 %p6, %rd39, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd18, %rd39, %rd17, %p6;
+; SM90-NEXT: selp.f64 %rd19, %rd2, %rd18, %p7;
+; SM90-NEXT: setp.eq.f64 %p8, %rd17, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd20, %rd19, %rd17, %p8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd38, %rd39};
+; SM90-NEXT: mov.b128 swap, {%rd15, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd21, %rd6, %rd39;
+; SM90-NEXT: xor.b64 %rd22, %rd5, %rd38;
+; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
+; SM90-NEXT: setp.ne.b64 %p9, %rd23, 0;
+; SM90-NEXT: mov.b64 %rd38, %rd5;
+; SM90-NEXT: mov.b64 %rd39, %rd6;
+; SM90-NEXT: @%p9 bra $L__BB211_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd24, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd24};
+; SM90-NEXT: mov.b128 swap, {%rd24, %rd24};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd40, %rd41}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p12, %rd3, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p16, %rd4, -9223372036854775808;
+; SM90-NEXT: $L__BB211_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p10, %rd40, %rd3;
+; SM90-NEXT: min.f64 %rd25, %rd40, %rd3;
+; SM90-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p10;
+; SM90-NEXT: setp.eq.b64 %p11, %rd40, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd27, %rd40, %rd26, %p11;
+; SM90-NEXT: selp.f64 %rd28, %rd3, %rd27, %p12;
+; SM90-NEXT: setp.eq.f64 %p13, %rd26, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd29, %rd28, %rd26, %p13;
+; SM90-NEXT: setp.nan.f64 %p14, %rd41, %rd4;
+; SM90-NEXT: min.f64 %rd30, %rd41, %rd4;
+; SM90-NEXT: selp.f64 %rd31, 0d7FF8000000000000, %rd30, %p14;
+; SM90-NEXT: setp.eq.b64 %p15, %rd41, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd32, %rd41, %rd31, %p15;
+; SM90-NEXT: selp.f64 %rd33, %rd4, %rd32, %p16;
+; SM90-NEXT: setp.eq.f64 %p17, %rd31, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd34, %rd33, %rd31, %p17;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd41};
+; SM90-NEXT: mov.b128 swap, {%rd29, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd35, %rd8, %rd41;
+; SM90-NEXT: xor.b64 %rd36, %rd7, %rd40;
+; SM90-NEXT: or.b64 %rd37, %rd36, %rd35;
+; SM90-NEXT: setp.ne.b64 %p18, %rd37, 0;
+; SM90-NEXT: mov.b64 %rd40, %rd7;
+; SM90-NEXT: mov.b64 %rd41, %rd8;
+; SM90-NEXT: @%p18 bra $L__BB211_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM90-LABEL: fminimum_acq_rel_v8f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<37>;
+; SM90-NEXT: .reg .b64 %rd<82>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd74, %rd75}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p3, %rd5, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p7, %rd6, -9223372036854775808;
+; SM90-NEXT: $L__BB212_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd75;
+; SM90-NEXT: mov.b64 %rd9, %rd74;
+; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd5;
+; SM90-NEXT: min.f64 %rd19, %rd9, %rd5;
+; SM90-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd9, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd21, %rd9, %rd20, %p2;
+; SM90-NEXT: selp.f64 %rd22, %rd5, %rd21, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd20, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd23, %rd22, %rd20, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd10, %rd6;
+; SM90-NEXT: min.f64 %rd24, %rd10, %rd6;
+; SM90-NEXT: selp.f64 %rd25, 0d7FF8000000000000, %rd24, %p5;
+; SM90-NEXT: setp.eq.b64 %p6, %rd10, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd26, %rd10, %rd25, %p6;
+; SM90-NEXT: selp.f64 %rd27, %rd6, %rd26, %p7;
+; SM90-NEXT: setp.eq.f64 %p8, %rd25, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd28, %rd27, %rd25, %p8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd23, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd74, %rd75}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd29, %rd75, %rd10;
+; SM90-NEXT: xor.b64 %rd30, %rd74, %rd9;
+; SM90-NEXT: or.b64 %rd31, %rd30, %rd29;
+; SM90-NEXT: setp.ne.b64 %p9, %rd31, 0;
+; SM90-NEXT: @%p9 bra $L__BB212_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd32, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd32, %rd32};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd32};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd76, %rd77}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p12, %rd7, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p16, %rd8, -9223372036854775808;
+; SM90-NEXT: $L__BB212_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd77;
+; SM90-NEXT: mov.b64 %rd11, %rd76;
+; SM90-NEXT: setp.nan.f64 %p10, %rd11, %rd7;
+; SM90-NEXT: min.f64 %rd33, %rd11, %rd7;
+; SM90-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p10;
+; SM90-NEXT: setp.eq.b64 %p11, %rd11, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd35, %rd11, %rd34, %p11;
+; SM90-NEXT: selp.f64 %rd36, %rd7, %rd35, %p12;
+; SM90-NEXT: setp.eq.f64 %p13, %rd34, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd37, %rd36, %rd34, %p13;
+; SM90-NEXT: setp.nan.f64 %p14, %rd12, %rd8;
+; SM90-NEXT: min.f64 %rd38, %rd12, %rd8;
+; SM90-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p14;
+; SM90-NEXT: setp.eq.b64 %p15, %rd12, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd40, %rd12, %rd39, %p15;
+; SM90-NEXT: selp.f64 %rd41, %rd8, %rd40, %p16;
+; SM90-NEXT: setp.eq.f64 %p17, %rd39, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd42, %rd41, %rd39, %p17;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd42};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd76, %rd77}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd43, %rd77, %rd12;
+; SM90-NEXT: xor.b64 %rd44, %rd76, %rd11;
+; SM90-NEXT: or.b64 %rd45, %rd44, %rd43;
+; SM90-NEXT: setp.ne.b64 %p18, %rd45, 0;
+; SM90-NEXT: @%p18 bra $L__BB212_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd46, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd46, %rd46};
+; SM90-NEXT: mov.b128 swap, {%rd46, %rd46};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd78, %rd79}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p21, %rd1, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p25, %rd2, -9223372036854775808;
+; SM90-NEXT: $L__BB212_5: // %atomicrmw.start15
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p19, %rd78, %rd1;
+; SM90-NEXT: min.f64 %rd47, %rd78, %rd1;
+; SM90-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p19;
+; SM90-NEXT: setp.eq.b64 %p20, %rd78, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd49, %rd78, %rd48, %p20;
+; SM90-NEXT: selp.f64 %rd50, %rd1, %rd49, %p21;
+; SM90-NEXT: setp.eq.f64 %p22, %rd48, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd51, %rd50, %rd48, %p22;
+; SM90-NEXT: setp.nan.f64 %p23, %rd79, %rd2;
+; SM90-NEXT: min.f64 %rd52, %rd79, %rd2;
+; SM90-NEXT: selp.f64 %rd53, 0d7FF8000000000000, %rd52, %p23;
+; SM90-NEXT: setp.eq.b64 %p24, %rd79, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd54, %rd79, %rd53, %p24;
+; SM90-NEXT: selp.f64 %rd55, %rd2, %rd54, %p25;
+; SM90-NEXT: setp.eq.f64 %p26, %rd53, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd56, %rd55, %rd53, %p26;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd78, %rd79};
+; SM90-NEXT: mov.b128 swap, {%rd51, %rd56};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd57, %rd14, %rd79;
+; SM90-NEXT: xor.b64 %rd58, %rd13, %rd78;
+; SM90-NEXT: or.b64 %rd59, %rd58, %rd57;
+; SM90-NEXT: setp.ne.b64 %p27, %rd59, 0;
+; SM90-NEXT: mov.b64 %rd78, %rd13;
+; SM90-NEXT: mov.b64 %rd79, %rd14;
+; SM90-NEXT: @%p27 bra $L__BB212_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end14
+; SM90-NEXT: mov.b64 %rd60, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd60, %rd60};
+; SM90-NEXT: mov.b128 swap, {%rd60, %rd60};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd80, %rd81}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p30, %rd3, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p34, %rd4, -9223372036854775808;
+; SM90-NEXT: $L__BB212_7: // %atomicrmw.start20
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p28, %rd80, %rd3;
+; SM90-NEXT: min.f64 %rd61, %rd80, %rd3;
+; SM90-NEXT: selp.f64 %rd62, 0d7FF8000000000000, %rd61, %p28;
+; SM90-NEXT: setp.eq.b64 %p29, %rd80, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd63, %rd80, %rd62, %p29;
+; SM90-NEXT: selp.f64 %rd64, %rd3, %rd63, %p30;
+; SM90-NEXT: setp.eq.f64 %p31, %rd62, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd65, %rd64, %rd62, %p31;
+; SM90-NEXT: setp.nan.f64 %p32, %rd81, %rd4;
+; SM90-NEXT: min.f64 %rd66, %rd81, %rd4;
+; SM90-NEXT: selp.f64 %rd67, 0d7FF8000000000000, %rd66, %p32;
+; SM90-NEXT: setp.eq.b64 %p33, %rd81, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd68, %rd81, %rd67, %p33;
+; SM90-NEXT: selp.f64 %rd69, %rd4, %rd68, %p34;
+; SM90-NEXT: setp.eq.f64 %p35, %rd67, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd70, %rd69, %rd67, %p35;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd80, %rd81};
+; SM90-NEXT: mov.b128 swap, {%rd65, %rd70};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd71, %rd16, %rd81;
+; SM90-NEXT: xor.b64 %rd72, %rd15, %rd80;
+; SM90-NEXT: or.b64 %rd73, %rd72, %rd71;
+; SM90-NEXT: setp.ne.b64 %p36, %rd73, 0;
+; SM90-NEXT: mov.b64 %rd80, %rd15;
+; SM90-NEXT: mov.b64 %rd81, %rd16;
+; SM90-NEXT: @%p36 bra $L__BB212_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end19
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd74, %rd75};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd76, %rd77};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v2f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b64 %rd<22>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd20, %rd21}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p3, %rd4, 0;
+; SM90-NEXT: setp.eq.b64 %p7, %rd5, 0;
+; SM90-NEXT: $L__BB213_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p1, %rd20, %rd4;
+; SM90-NEXT: max.f64 %rd7, %rd20, %rd4;
+; SM90-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd20, 0;
+; SM90-NEXT: selp.f64 %rd9, %rd20, %rd8, %p2;
+; SM90-NEXT: selp.f64 %rd10, %rd4, %rd9, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd8, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd11, %rd10, %rd8, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd21, %rd5;
+; SM90-NEXT: max.f64 %rd12, %rd21, %rd5;
+; SM90-NEXT: selp.f64 %rd13, 0d7FF8000000000000, %rd12, %p5;
+; SM90-NEXT: setp.eq.b64 %p6, %rd21, 0;
+; SM90-NEXT: selp.f64 %rd14, %rd21, %rd13, %p6;
+; SM90-NEXT: selp.f64 %rd15, %rd5, %rd14, %p7;
+; SM90-NEXT: setp.eq.f64 %p8, %rd13, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd16, %rd15, %rd13, %p8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd20, %rd21};
+; SM90-NEXT: mov.b128 swap, {%rd11, %rd16};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd17, %rd2, %rd21;
+; SM90-NEXT: xor.b64 %rd18, %rd1, %rd20;
+; SM90-NEXT: or.b64 %rd19, %rd18, %rd17;
+; SM90-NEXT: setp.ne.b64 %p9, %rd19, 0;
+; SM90-NEXT: mov.b64 %rd20, %rd1;
+; SM90-NEXT: mov.b64 %rd21, %rd2;
+; SM90-NEXT: @%p9 bra $L__BB213_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v4f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<19>;
+; SM90-NEXT: .reg .b64 %rd<42>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd38, %rd39}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM90-NEXT: setp.eq.b64 %p7, %rd2, 0;
+; SM90-NEXT: $L__BB214_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p1, %rd38, %rd1;
+; SM90-NEXT: max.f64 %rd11, %rd38, %rd1;
+; SM90-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd38, 0;
+; SM90-NEXT: selp.f64 %rd13, %rd38, %rd12, %p2;
+; SM90-NEXT: selp.f64 %rd14, %rd1, %rd13, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd12, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd15, %rd14, %rd12, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd39, %rd2;
+; SM90-NEXT: max.f64 %rd16, %rd39, %rd2;
+; SM90-NEXT: selp.f64 %rd17, 0d7FF8000000000000, %rd16, %p5;
+; SM90-NEXT: setp.eq.b64 %p6, %rd39, 0;
+; SM90-NEXT: selp.f64 %rd18, %rd39, %rd17, %p6;
+; SM90-NEXT: selp.f64 %rd19, %rd2, %rd18, %p7;
+; SM90-NEXT: setp.eq.f64 %p8, %rd17, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd20, %rd19, %rd17, %p8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd38, %rd39};
+; SM90-NEXT: mov.b128 swap, {%rd15, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd21, %rd6, %rd39;
+; SM90-NEXT: xor.b64 %rd22, %rd5, %rd38;
+; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
+; SM90-NEXT: setp.ne.b64 %p9, %rd23, 0;
+; SM90-NEXT: mov.b64 %rd38, %rd5;
+; SM90-NEXT: mov.b64 %rd39, %rd6;
+; SM90-NEXT: @%p9 bra $L__BB214_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd24, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd24};
+; SM90-NEXT: mov.b128 swap, {%rd24, %rd24};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd40, %rd41}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p12, %rd3, 0;
+; SM90-NEXT: setp.eq.b64 %p16, %rd4, 0;
+; SM90-NEXT: $L__BB214_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p10, %rd40, %rd3;
+; SM90-NEXT: max.f64 %rd25, %rd40, %rd3;
+; SM90-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p10;
+; SM90-NEXT: setp.eq.b64 %p11, %rd40, 0;
+; SM90-NEXT: selp.f64 %rd27, %rd40, %rd26, %p11;
+; SM90-NEXT: selp.f64 %rd28, %rd3, %rd27, %p12;
+; SM90-NEXT: setp.eq.f64 %p13, %rd26, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd29, %rd28, %rd26, %p13;
+; SM90-NEXT: setp.nan.f64 %p14, %rd41, %rd4;
+; SM90-NEXT: max.f64 %rd30, %rd41, %rd4;
+; SM90-NEXT: selp.f64 %rd31, 0d7FF8000000000000, %rd30, %p14;
+; SM90-NEXT: setp.eq.b64 %p15, %rd41, 0;
+; SM90-NEXT: selp.f64 %rd32, %rd41, %rd31, %p15;
+; SM90-NEXT: selp.f64 %rd33, %rd4, %rd32, %p16;
+; SM90-NEXT: setp.eq.f64 %p17, %rd31, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd34, %rd33, %rd31, %p17;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd41};
+; SM90-NEXT: mov.b128 swap, {%rd29, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd35, %rd8, %rd41;
+; SM90-NEXT: xor.b64 %rd36, %rd7, %rd40;
+; SM90-NEXT: or.b64 %rd37, %rd36, %rd35;
+; SM90-NEXT: setp.ne.b64 %p18, %rd37, 0;
+; SM90-NEXT: mov.b64 %rd40, %rd7;
+; SM90-NEXT: mov.b64 %rd41, %rd8;
+; SM90-NEXT: @%p18 bra $L__BB214_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v8f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<37>;
+; SM90-NEXT: .reg .b64 %rd<82>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd74, %rd75}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p3, %rd5, 0;
+; SM90-NEXT: setp.eq.b64 %p7, %rd6, 0;
+; SM90-NEXT: $L__BB215_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd75;
+; SM90-NEXT: mov.b64 %rd9, %rd74;
+; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd5;
+; SM90-NEXT: max.f64 %rd19, %rd9, %rd5;
+; SM90-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd9, 0;
+; SM90-NEXT: selp.f64 %rd21, %rd9, %rd20, %p2;
+; SM90-NEXT: selp.f64 %rd22, %rd5, %rd21, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd20, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd23, %rd22, %rd20, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd10, %rd6;
+; SM90-NEXT: max.f64 %rd24, %rd10, %rd6;
+; SM90-NEXT: selp.f64 %rd25, 0d7FF8000000000000, %rd24, %p5;
+; SM90-NEXT: setp.eq.b64 %p6, %rd10, 0;
+; SM90-NEXT: selp.f64 %rd26, %rd10, %rd25, %p6;
+; SM90-NEXT: selp.f64 %rd27, %rd6, %rd26, %p7;
+; SM90-NEXT: setp.eq.f64 %p8, %rd25, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd28, %rd27, %rd25, %p8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd23, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd74, %rd75}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd29, %rd75, %rd10;
+; SM90-NEXT: xor.b64 %rd30, %rd74, %rd9;
+; SM90-NEXT: or.b64 %rd31, %rd30, %rd29;
+; SM90-NEXT: setp.ne.b64 %p9, %rd31, 0;
+; SM90-NEXT: @%p9 bra $L__BB215_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd32, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd32, %rd32};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd32};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd76, %rd77}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p12, %rd7, 0;
+; SM90-NEXT: setp.eq.b64 %p16, %rd8, 0;
+; SM90-NEXT: $L__BB215_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd77;
+; SM90-NEXT: mov.b64 %rd11, %rd76;
+; SM90-NEXT: setp.nan.f64 %p10, %rd11, %rd7;
+; SM90-NEXT: max.f64 %rd33, %rd11, %rd7;
+; SM90-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p10;
+; SM90-NEXT: setp.eq.b64 %p11, %rd11, 0;
+; SM90-NEXT: selp.f64 %rd35, %rd11, %rd34, %p11;
+; SM90-NEXT: selp.f64 %rd36, %rd7, %rd35, %p12;
+; SM90-NEXT: setp.eq.f64 %p13, %rd34, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd37, %rd36, %rd34, %p13;
+; SM90-NEXT: setp.nan.f64 %p14, %rd12, %rd8;
+; SM90-NEXT: max.f64 %rd38, %rd12, %rd8;
+; SM90-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p14;
+; SM90-NEXT: setp.eq.b64 %p15, %rd12, 0;
+; SM90-NEXT: selp.f64 %rd40, %rd12, %rd39, %p15;
+; SM90-NEXT: selp.f64 %rd41, %rd8, %rd40, %p16;
+; SM90-NEXT: setp.eq.f64 %p17, %rd39, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd42, %rd41, %rd39, %p17;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd42};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd76, %rd77}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd43, %rd77, %rd12;
+; SM90-NEXT: xor.b64 %rd44, %rd76, %rd11;
+; SM90-NEXT: or.b64 %rd45, %rd44, %rd43;
+; SM90-NEXT: setp.ne.b64 %p18, %rd45, 0;
+; SM90-NEXT: @%p18 bra $L__BB215_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd46, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd46, %rd46};
+; SM90-NEXT: mov.b128 swap, {%rd46, %rd46};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd78, %rd79}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p21, %rd1, 0;
+; SM90-NEXT: setp.eq.b64 %p25, %rd2, 0;
+; SM90-NEXT: $L__BB215_5: // %atomicrmw.start15
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p19, %rd78, %rd1;
+; SM90-NEXT: max.f64 %rd47, %rd78, %rd1;
+; SM90-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p19;
+; SM90-NEXT: setp.eq.b64 %p20, %rd78, 0;
+; SM90-NEXT: selp.f64 %rd49, %rd78, %rd48, %p20;
+; SM90-NEXT: selp.f64 %rd50, %rd1, %rd49, %p21;
+; SM90-NEXT: setp.eq.f64 %p22, %rd48, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd51, %rd50, %rd48, %p22;
+; SM90-NEXT: setp.nan.f64 %p23, %rd79, %rd2;
+; SM90-NEXT: max.f64 %rd52, %rd79, %rd2;
+; SM90-NEXT: selp.f64 %rd53, 0d7FF8000000000000, %rd52, %p23;
+; SM90-NEXT: setp.eq.b64 %p24, %rd79, 0;
+; SM90-NEXT: selp.f64 %rd54, %rd79, %rd53, %p24;
+; SM90-NEXT: selp.f64 %rd55, %rd2, %rd54, %p25;
+; SM90-NEXT: setp.eq.f64 %p26, %rd53, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd56, %rd55, %rd53, %p26;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd78, %rd79};
+; SM90-NEXT: mov.b128 swap, {%rd51, %rd56};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd57, %rd14, %rd79;
+; SM90-NEXT: xor.b64 %rd58, %rd13, %rd78;
+; SM90-NEXT: or.b64 %rd59, %rd58, %rd57;
+; SM90-NEXT: setp.ne.b64 %p27, %rd59, 0;
+; SM90-NEXT: mov.b64 %rd78, %rd13;
+; SM90-NEXT: mov.b64 %rd79, %rd14;
+; SM90-NEXT: @%p27 bra $L__BB215_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end14
+; SM90-NEXT: mov.b64 %rd60, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd60, %rd60};
+; SM90-NEXT: mov.b128 swap, {%rd60, %rd60};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd80, %rd81}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p30, %rd3, 0;
+; SM90-NEXT: setp.eq.b64 %p34, %rd4, 0;
+; SM90-NEXT: $L__BB215_7: // %atomicrmw.start20
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p28, %rd80, %rd3;
+; SM90-NEXT: max.f64 %rd61, %rd80, %rd3;
+; SM90-NEXT: selp.f64 %rd62, 0d7FF8000000000000, %rd61, %p28;
+; SM90-NEXT: setp.eq.b64 %p29, %rd80, 0;
+; SM90-NEXT: selp.f64 %rd63, %rd80, %rd62, %p29;
+; SM90-NEXT: selp.f64 %rd64, %rd3, %rd63, %p30;
+; SM90-NEXT: setp.eq.f64 %p31, %rd62, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd65, %rd64, %rd62, %p31;
+; SM90-NEXT: setp.nan.f64 %p32, %rd81, %rd4;
+; SM90-NEXT: max.f64 %rd66, %rd81, %rd4;
+; SM90-NEXT: selp.f64 %rd67, 0d7FF8000000000000, %rd66, %p32;
+; SM90-NEXT: setp.eq.b64 %p33, %rd81, 0;
+; SM90-NEXT: selp.f64 %rd68, %rd81, %rd67, %p33;
+; SM90-NEXT: selp.f64 %rd69, %rd4, %rd68, %p34;
+; SM90-NEXT: setp.eq.f64 %p35, %rd67, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd70, %rd69, %rd67, %p35;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd80, %rd81};
+; SM90-NEXT: mov.b128 swap, {%rd65, %rd70};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd71, %rd16, %rd81;
+; SM90-NEXT: xor.b64 %rd72, %rd15, %rd80;
+; SM90-NEXT: or.b64 %rd73, %rd72, %rd71;
+; SM90-NEXT: setp.ne.b64 %p36, %rd73, 0;
+; SM90-NEXT: mov.b64 %rd80, %rd15;
+; SM90-NEXT: mov.b64 %rd81, %rd16;
+; SM90-NEXT: @%p36 bra $L__BB215_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end19
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd74, %rd75};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd76, %rd77};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fadd_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fadd_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs5, [%rd1], %rs1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs2;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs3;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fadd_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<17>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs9, [%rd1], %rs7;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+2], %rs8;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+4], %rs5;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs6;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs13, [%rd1+8], %rs3;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+10], %rs4;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: mov.b32 %r5, {%rs9, %rs10};
+; SM90-NEXT: mov.b32 %r6, {%rs11, %rs12};
+; SM90-NEXT: mov.b32 %r7, {%rs13, %rs14};
+; SM90-NEXT: mov.b32 %r8, {%rs15, %rs16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fsub_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB219_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB219_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fsub_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB220_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: sub.rn.f16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB220_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fsub_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB221_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: sub.rn.f16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: sub.rn.f16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: sub.rn.f16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB221_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fmin_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB222_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB222_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fmin_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB223_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: min.f16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB223_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fmin_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB224_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: min.f16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.f16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: min.f16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB224_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB225_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB225_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB226_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: max.f16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB226_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB227_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: max.f16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.f16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: max.f16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB227_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fminimum_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB228_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB228_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fminimum_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB229_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: min.NaN.f16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB229_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fminimum_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB230_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: min.NaN.f16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.NaN.f16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: min.NaN.f16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB230_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB231_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB231_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB232_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: max.NaN.f16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB232_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB233_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: max.NaN.f16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.NaN.f16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: max.NaN.f16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB233_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM90-LABEL: fadd_acq_rel_v2bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs3, [%rd1], %rs1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs4, [%rd1+2], %rs2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM90-LABEL: fadd_acq_rel_v4bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs5, [%rd1], %rs1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs6, [%rd1+2], %rs2;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs7, [%rd1+4], %rs3;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs8, [%rd1+6], %rs4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM90-LABEL: fadd_acq_rel_v8bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<17>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs9, [%rd1], %rs7;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs10, [%rd1+2], %rs8;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs11, [%rd1+4], %rs5;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs12, [%rd1+6], %rs6;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs13, [%rd1+8], %rs3;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs14, [%rd1+10], %rs4;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs15, [%rd1+12], %rs1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs16, [%rd1+14], %rs2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: mov.b32 %r5, {%rs9, %rs10};
+; SM90-NEXT: mov.b32 %r6, {%rs11, %rs12};
+; SM90-NEXT: mov.b32 %r7, {%rs13, %rs14};
+; SM90-NEXT: mov.b32 %r8, {%rs15, %rs16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM90-LABEL: fsub_acq_rel_v2bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB237_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.bf16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB237_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM90-LABEL: fsub_acq_rel_v4bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB238_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.bf16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: sub.rn.bf16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB238_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM90-LABEL: fsub_acq_rel_v8bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB239_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.bf16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: sub.rn.bf16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: sub.rn.bf16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: sub.rn.bf16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB239_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM90-LABEL: fmin_acq_rel_v2bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB240_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.bf16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB240_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM90-LABEL: fmin_acq_rel_v4bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB241_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.bf16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: min.bf16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB241_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM90-LABEL: fmin_acq_rel_v8bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB242_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.bf16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: min.bf16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.bf16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: min.bf16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB242_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM90-LABEL: fmax_acq_rel_v2bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB243_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.bf16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB243_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM90-LABEL: fmax_acq_rel_v4bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB244_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.bf16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: max.bf16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB244_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM90-LABEL: fmax_acq_rel_v8bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB245_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.bf16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: max.bf16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.bf16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: max.bf16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB245_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM90-LABEL: fminimum_acq_rel_v2bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB246_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.bf16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB246_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM90-LABEL: fminimum_acq_rel_v4bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB247_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.bf16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: min.NaN.bf16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB247_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM90-LABEL: fminimum_acq_rel_v8bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB248_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.bf16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: min.NaN.bf16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.NaN.bf16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: min.NaN.bf16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB248_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB249_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.bf16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB249_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v4bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB250_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.bf16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: max.NaN.bf16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB250_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v8bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB251_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.bf16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: max.NaN.bf16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.NaN.bf16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: max.NaN.bf16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB251_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: add_monotonic_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<10>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_monotonic_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [add_monotonic_v2i8_global_cta_param_0];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM90-NEXT: $L__BB252_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: and.b16 %rs8, %rs5, 255;
+; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: shl.b32 %r13, %r12, %r1;
+; SM90-NEXT: and.b32 %r14, %r17, %r2;
+; SM90-NEXT: or.b32 %r15, %r14, %r13;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM90-NEXT: mov.b32 %r17, %r3;
+; SM90-NEXT: @%p1 bra $L__BB252_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r16, %r3, %r1;
+; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: add_acquire_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<10>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM90-NEXT: $L__BB253_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: and.b16 %rs8, %rs5, 255;
+; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: shl.b32 %r13, %r12, %r1;
+; SM90-NEXT: and.b32 %r14, %r17, %r2;
+; SM90-NEXT: or.b32 %r15, %r14, %r13;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM90-NEXT: mov.b32 %r17, %r3;
+; SM90-NEXT: @%p1 bra $L__BB253_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r16, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: add_release_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<10>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM90-NEXT: $L__BB254_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: and.b16 %rs8, %rs5, 255;
+; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: shl.b32 %r13, %r12, %r1;
+; SM90-NEXT: and.b32 %r14, %r17, %r2;
+; SM90-NEXT: or.b32 %r15, %r14, %r13;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM90-NEXT: mov.b32 %r17, %r3;
+; SM90-NEXT: @%p1 bra $L__BB254_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r16, %r3, %r1;
+; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: add_seq_cst_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<10>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM90-NEXT: $L__BB255_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: and.b16 %rs8, %rs5, 255;
+; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: shl.b32 %r13, %r12, %r1;
+; SM90-NEXT: and.b32 %r14, %r17, %r2;
+; SM90-NEXT: or.b32 %r15, %r14, %r13;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM90-NEXT: mov.b32 %r17, %r3;
+; SM90-NEXT: @%p1 bra $L__BB255_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r16, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_monotonic_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [add_monotonic_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: $L__BB256_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB256_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_acquire_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: $L__BB257_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB257_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_release_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: $L__BB258_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB258_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_seq_cst_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: $L__BB259_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB259_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: add_monotonic_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB260_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB260_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: add_acquire_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB261_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB261_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: add_release_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB262_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB262_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: add_seq_cst_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB263_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB263_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
+}
+
+define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: add_monotonic_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: add_acquire_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: add_release_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: add_seq_cst_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: add_monotonic_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: add_acquire_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: add_release_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: add_seq_cst_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: add_monotonic_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: add_acquire_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: add_release_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_release_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: add_seq_cst_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_seq_cst_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
+ ret <8 x i32> %retval
+}
+
+define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: nand_monotonic_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_monotonic_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [nand_monotonic_v2i8_global_cta_param_0];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB276_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, %r4;
+; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB276_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: nand_acquire_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acquire_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [nand_acquire_v2i8_global_cta_param_0];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB277_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, %r4;
+; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB277_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: nand_release_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB278_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, %r4;
+; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB278_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: nand_seq_cst_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB279_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, %r4;
+; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB279_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: nand_monotonic_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB280_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB280_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: nand_acquire_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB281_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB281_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: nand_release_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB282_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB282_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
+}
+
+define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: nand_seq_cst_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [nand_seq_cst_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB283_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB283_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_monotonic_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB284_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB284_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_acquire_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB285_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB285_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_release_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB286_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB286_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
+ ret <8 x i8> %retval
+}
+
+define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_seq_cst_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB287_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB287_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
+}
+
+define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_monotonic_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB288_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB288_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_acquire_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB289_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB289_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_release_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB290_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB290_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_seq_cst_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB291_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB291_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_monotonic_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB292_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r12, %r4;
+; SM90-NEXT: and.b32 %r6, %r11, %r3;
+; SM90-NEXT: and.b32 %r7, %r10, %r2;
+; SM90-NEXT: and.b32 %r8, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB292_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_acquire_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB293_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r12, %r4;
+; SM90-NEXT: and.b32 %r6, %r11, %r3;
+; SM90-NEXT: and.b32 %r7, %r10, %r2;
+; SM90-NEXT: and.b32 %r8, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB293_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_release_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB294_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r12, %r4;
+; SM90-NEXT: and.b32 %r6, %r11, %r3;
+; SM90-NEXT: and.b32 %r7, %r10, %r2;
+; SM90-NEXT: and.b32 %r8, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB294_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_seq_cst_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB295_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r12, %r4;
+; SM90-NEXT: and.b32 %r6, %r11, %r3;
+; SM90-NEXT: and.b32 %r7, %r10, %r2;
+; SM90-NEXT: and.b32 %r8, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB295_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: nand_monotonic_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<54>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_monotonic_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB296_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r20;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r9, %r20, %r4;
+; SM90-NEXT: and.b32 %r10, %r19, %r3;
+; SM90-NEXT: and.b32 %r11, %r18, %r2;
+; SM90-NEXT: and.b32 %r12, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r12;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r10;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r9;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd24;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB296_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd28, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd29, %rd30}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd30;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
+; SM90-NEXT: $L__BB296_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
+; SM90-NEXT: shl.b64 %rd33, %rd32, 32;
+; SM90-NEXT: or.b64 %rd34, %rd31, %rd33;
+; SM90-NEXT: cvt.u64.u32 %rd35, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd36, %r24;
+; SM90-NEXT: shl.b64 %rd37, %rd36, 32;
+; SM90-NEXT: or.b64 %rd38, %rd35, %rd37;
+; SM90-NEXT: and.b32 %r13, %r24, %r8;
+; SM90-NEXT: and.b32 %r14, %r23, %r7;
+; SM90-NEXT: and.b32 %r15, %r22, %r6;
+; SM90-NEXT: and.b32 %r16, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd39, %r16;
+; SM90-NEXT: cvt.u64.u32 %rd40, %r15;
+; SM90-NEXT: shl.b64 %rd41, %rd40, 32;
+; SM90-NEXT: or.b64 %rd42, %rd39, %rd41;
+; SM90-NEXT: not.b64 %rd43, %rd42;
+; SM90-NEXT: cvt.u64.u32 %rd44, %r14;
+; SM90-NEXT: cvt.u64.u32 %rd45, %r13;
+; SM90-NEXT: shl.b64 %rd46, %rd45, 32;
+; SM90-NEXT: or.b64 %rd47, %rd44, %rd46;
+; SM90-NEXT: not.b64 %rd48, %rd47;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd38};
+; SM90-NEXT: mov.b128 swap, {%rd43, %rd48};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd49, %rd50}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd51, %rd50, %rd38;
+; SM90-NEXT: xor.b64 %rd52, %rd49, %rd34;
+; SM90-NEXT: or.b64 %rd53, %rd52, %rd51;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd50;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd50;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
+; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
+; SM90-NEXT: @%p2 bra $L__BB296_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: nand_acquire_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<54>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acquire_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r20;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r9, %r20, %r4;
+; SM90-NEXT: and.b32 %r10, %r19, %r3;
+; SM90-NEXT: and.b32 %r11, %r18, %r2;
+; SM90-NEXT: and.b32 %r12, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r12;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r10;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r9;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd24;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB297_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd28, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd29, %rd30}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd30;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
+; SM90-NEXT: $L__BB297_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
+; SM90-NEXT: shl.b64 %rd33, %rd32, 32;
+; SM90-NEXT: or.b64 %rd34, %rd31, %rd33;
+; SM90-NEXT: cvt.u64.u32 %rd35, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd36, %r24;
+; SM90-NEXT: shl.b64 %rd37, %rd36, 32;
+; SM90-NEXT: or.b64 %rd38, %rd35, %rd37;
+; SM90-NEXT: and.b32 %r13, %r24, %r8;
+; SM90-NEXT: and.b32 %r14, %r23, %r7;
+; SM90-NEXT: and.b32 %r15, %r22, %r6;
+; SM90-NEXT: and.b32 %r16, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd39, %r16;
+; SM90-NEXT: cvt.u64.u32 %rd40, %r15;
+; SM90-NEXT: shl.b64 %rd41, %rd40, 32;
+; SM90-NEXT: or.b64 %rd42, %rd39, %rd41;
+; SM90-NEXT: not.b64 %rd43, %rd42;
+; SM90-NEXT: cvt.u64.u32 %rd44, %r14;
+; SM90-NEXT: cvt.u64.u32 %rd45, %r13;
+; SM90-NEXT: shl.b64 %rd46, %rd45, 32;
+; SM90-NEXT: or.b64 %rd47, %rd44, %rd46;
+; SM90-NEXT: not.b64 %rd48, %rd47;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd38};
+; SM90-NEXT: mov.b128 swap, {%rd43, %rd48};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd49, %rd50}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd51, %rd50, %rd38;
+; SM90-NEXT: xor.b64 %rd52, %rd49, %rd34;
+; SM90-NEXT: or.b64 %rd53, %rd52, %rd51;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd50;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd50;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
+; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
+; SM90-NEXT: @%p2 bra $L__BB297_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: nand_release_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<54>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_release_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB298_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r20;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r9, %r20, %r4;
+; SM90-NEXT: and.b32 %r10, %r19, %r3;
+; SM90-NEXT: and.b32 %r11, %r18, %r2;
+; SM90-NEXT: and.b32 %r12, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r12;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r10;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r9;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd24;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB298_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd28, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd29, %rd30}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd30;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
+; SM90-NEXT: $L__BB298_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
+; SM90-NEXT: shl.b64 %rd33, %rd32, 32;
+; SM90-NEXT: or.b64 %rd34, %rd31, %rd33;
+; SM90-NEXT: cvt.u64.u32 %rd35, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd36, %r24;
+; SM90-NEXT: shl.b64 %rd37, %rd36, 32;
+; SM90-NEXT: or.b64 %rd38, %rd35, %rd37;
+; SM90-NEXT: and.b32 %r13, %r24, %r8;
+; SM90-NEXT: and.b32 %r14, %r23, %r7;
+; SM90-NEXT: and.b32 %r15, %r22, %r6;
+; SM90-NEXT: and.b32 %r16, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd39, %r16;
+; SM90-NEXT: cvt.u64.u32 %rd40, %r15;
+; SM90-NEXT: shl.b64 %rd41, %rd40, 32;
+; SM90-NEXT: or.b64 %rd42, %rd39, %rd41;
+; SM90-NEXT: not.b64 %rd43, %rd42;
+; SM90-NEXT: cvt.u64.u32 %rd44, %r14;
+; SM90-NEXT: cvt.u64.u32 %rd45, %r13;
+; SM90-NEXT: shl.b64 %rd46, %rd45, 32;
+; SM90-NEXT: or.b64 %rd47, %rd44, %rd46;
+; SM90-NEXT: not.b64 %rd48, %rd47;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd38};
+; SM90-NEXT: mov.b128 swap, {%rd43, %rd48};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd49, %rd50}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd51, %rd50, %rd38;
+; SM90-NEXT: xor.b64 %rd52, %rd49, %rd34;
+; SM90-NEXT: or.b64 %rd53, %rd52, %rd51;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd50;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd50;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
+; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
+; SM90-NEXT: @%p2 bra $L__BB298_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
+ ret <8 x i32> %retval
+}
+
+define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: nand_seq_cst_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<54>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_seq_cst_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB299_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r20;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r9, %r20, %r4;
+; SM90-NEXT: and.b32 %r10, %r19, %r3;
+; SM90-NEXT: and.b32 %r11, %r18, %r2;
+; SM90-NEXT: and.b32 %r12, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r12;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r10;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r9;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd24;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB299_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd28, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd29, %rd30}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd30;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
+; SM90-NEXT: $L__BB299_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
+; SM90-NEXT: shl.b64 %rd33, %rd32, 32;
+; SM90-NEXT: or.b64 %rd34, %rd31, %rd33;
+; SM90-NEXT: cvt.u64.u32 %rd35, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd36, %r24;
+; SM90-NEXT: shl.b64 %rd37, %rd36, 32;
+; SM90-NEXT: or.b64 %rd38, %rd35, %rd37;
+; SM90-NEXT: and.b32 %r13, %r24, %r8;
+; SM90-NEXT: and.b32 %r14, %r23, %r7;
+; SM90-NEXT: and.b32 %r15, %r22, %r6;
+; SM90-NEXT: and.b32 %r16, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd39, %r16;
+; SM90-NEXT: cvt.u64.u32 %rd40, %r15;
+; SM90-NEXT: shl.b64 %rd41, %rd40, 32;
+; SM90-NEXT: or.b64 %rd42, %rd39, %rd41;
+; SM90-NEXT: not.b64 %rd43, %rd42;
+; SM90-NEXT: cvt.u64.u32 %rd44, %r14;
+; SM90-NEXT: cvt.u64.u32 %rd45, %r13;
+; SM90-NEXT: shl.b64 %rd46, %rd45, 32;
+; SM90-NEXT: or.b64 %rd47, %rd44, %rd46;
+; SM90-NEXT: not.b64 %rd48, %rd47;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd38};
+; SM90-NEXT: mov.b128 swap, {%rd43, %rd48};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd49, %rd50}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd51, %rd50, %rd38;
+; SM90-NEXT: xor.b64 %rd52, %rd49, %rd34;
+; SM90-NEXT: or.b64 %rd53, %rd52, %rd51;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd50;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd50;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
+; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
+; SM90-NEXT: @%p2 bra $L__BB299_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
+ ret <8 x i32> %retval
+}
+
diff --git a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
new file mode 100644
index 0000000000000..7315a456f12c1
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
@@ -0,0 +1,162 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=nvptx64-nvidia-cuda -mcpu=sm_90 -mattr=+ptx83 | FileCheck %s
+
+target triple = "nvptx64-nvidia-cuda"
+
+; Elementwise expansion: scalar f32 atomics are legal, so the vector is split
+; into scalar atomicrmw instructions.
+define <2 x float> @fadd_v2f32_elementwise(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: @fadd_v2f32_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LO_VAL:%.*]] = extractelement <2 x float> [[VAL:%.*]], i64 0
+; CHECK-NEXT: [[HI_VAL:%.*]] = extractelement <2 x float> [[VAL]], i64 1
+; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds float, ptr [[ADDR:%.*]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR]], float [[LO_VAL]] monotonic, align 8
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr [[HI_PTR]], float [[HI_VAL]] monotonic, align 4
+; CHECK-NEXT: [[LO_OLD:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
+; CHECK-NEXT: [[HI_OLD:%.*]] = insertelement <2 x float> [[LO_OLD]], float [[TMP1]], i64 1
+; CHECK-NEXT: ret <2 x float> [[HI_OLD]]
+;
+entry:
+ %old = atomicrmw elementwise fadd ptr %addr, <2 x float> %val monotonic
+ ret <2 x float> %old
+}
+
+; Ordered elementwise expansion: fences should be inserted around the whole
+; split expansion rather than around each split child.
+define <4 x i32> @add_acq_rel_v4i32_elementwise(ptr %addr, <4 x i32> %val) {
+; CHECK-LABEL: @add_acq_rel_v4i32_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: fence syncscope("block") release
+; CHECK-NEXT: [[LO_VAL:%.*]] = shufflevector <4 x i32> [[VAL:%.*]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[HI_VAL:%.*]] = shufflevector <4 x i32> [[VAL]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds <2 x i32>, ptr [[ADDR:%.*]], i64 1
+; CHECK-NEXT: [[LO_VAL2:%.*]] = extractelement <2 x i32> [[LO_VAL]], i64 0
+; CHECK-NEXT: [[HI_VAL3:%.*]] = extractelement <2 x i32> [[LO_VAL]], i64 1
+; CHECK-NEXT: [[HI_PTR4:%.*]] = getelementptr inbounds i32, ptr [[ADDR]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[LO_VAL2]] syncscope("block") monotonic, align 16
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[HI_PTR4]], i32 [[HI_VAL3]] syncscope("block") monotonic, align 4
+; CHECK-NEXT: [[LO_OLD:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[HI_OLD:%.*]] = insertelement <2 x i32> [[LO_OLD]], i32 [[TMP1]], i64 1
+; CHECK-NEXT: [[LO_VAL5:%.*]] = extractelement <2 x i32> [[HI_VAL]], i64 0
+; CHECK-NEXT: [[HI_VAL6:%.*]] = extractelement <2 x i32> [[HI_VAL]], i64 1
+; CHECK-NEXT: [[HI_PTR7:%.*]] = getelementptr inbounds i32, ptr [[HI_PTR]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = atomicrmw add ptr [[HI_PTR]], i32 [[LO_VAL5]] syncscope("block") monotonic, align 8
+; CHECK-NEXT: [[TMP3:%.*]] = atomicrmw add ptr [[HI_PTR7]], i32 [[HI_VAL6]] syncscope("block") monotonic, align 4
+; CHECK-NEXT: [[LO_OLD8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP2]], i64 0
+; CHECK-NEXT: [[HI_OLD9:%.*]] = insertelement <2 x i32> [[LO_OLD8]], i32 [[TMP3]], i64 1
+; CHECK-NEXT: [[OLD1:%.*]] = shufflevector <2 x i32> [[HI_OLD]], <2 x i32> [[HI_OLD9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: fence syncscope("block") acquire
+; CHECK-NEXT: ret <4 x i32> [[OLD1]]
+;
+entry:
+ %old = atomicrmw elementwise add ptr %addr, <4 x i32> %val syncscope("block") acq_rel, align 16
+ ret <4 x i32> %old
+}
+
+; Whole-vector cmpxchg expansion: sm_90 + ptx83 supports 128-bit cmpxchg, so
+; this vector can stay whole during AtomicExpand.
+define <8 x half> @fmin_v8f16_elementwise(ptr %addr, <8 x half> %val) {
+; CHECK-LABEL: @fmin_v8f16_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load atomic i128, ptr [[ADDR:%.*]] monotonic, align 16
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i128 [[TMP0]] to <8 x half>
+; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
+; CHECK: atomicrmw.start:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <8 x half> [ [[TMP1]], [[ENTRY:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <8 x half> @llvm.minnum.v8f16(<8 x half> [[LOADED]], <8 x half> [[VAL:%.*]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x half> [[TMP2]] to i128
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x half> [[LOADED]] to i128
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i128 [[TMP4]], i128 [[TMP3]] monotonic monotonic, align 16
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i128, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i128 [[NEWLOADED]] to <8 x half>
+; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK: atomicrmw.end:
+; CHECK-NEXT: ret <8 x half> [[TMP6]]
+;
+entry:
+ %old = atomicrmw elementwise fmin ptr %addr, <8 x half> %val monotonic, align 16
+ ret <8 x half> %old
+}
+
+; Vector xchg is bit-equivalent to xchg on the same-width integer type.
+define <4 x i8> @xchg_v4i8_elementwise(ptr %addr, <4 x i8> %val) {
+; CHECK-LABEL: @xchg_v4i8_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i8> [[VAL:%.*]] to i32
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw xchg ptr [[ADDR:%.*]], i32 [[TMP0]] monotonic, align 4
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to <4 x i8>
+; CHECK-NEXT: ret <4 x i8> [[TMP2]]
+;
+entry:
+ %old = atomicrmw elementwise xchg ptr %addr, <4 x i8> %val monotonic, align 4
+ ret <4 x i8> %old
+}
+
+; Partword bitwise vector atomics must bitcast before widening; zext directly
+; from <2 x i8> to i32 is invalid IR.
+define <2 x i8> @and_v2i8_partword_elementwise(ptr %addr, <2 x i8> %val) {
+; CHECK-LABEL: @and_v2i8_partword_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[ADDR:%.*]], i64 -4)
+; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[ADDR]] to i64
+; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[PTRLSB]], 3
+; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i8> [[VAL:%.*]] to i16
+; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; CHECK-NEXT: [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; CHECK-NEXT: [[TMP4:%.*]] = atomicrmw and ptr [[ALIGNEDADDR]], i32 [[ANDOPERAND]] monotonic, align 4
+; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED]] to <2 x i8>
+; CHECK-NEXT: ret <2 x i8> [[TMP5]]
+;
+entry:
+ %old = atomicrmw elementwise and ptr %addr, <2 x i8> %val monotonic
+ ret <2 x i8> %old
+}
+
+; Partword vector arithmetic must operate lane-wise before reinserting into the
+; containing CAS word, otherwise packed integer carries can cross lanes.
+define <2 x i8> @add_v2i8_partword_elementwise(ptr %addr, <2 x i8> %val) {
+; CHECK-LABEL: @add_v2i8_partword_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[ADDR:%.*]], i64 -4)
+; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[ADDR]] to i64
+; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[PTRLSB]], 3
+; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; CHECK-NEXT: [[TMP2:%.*]] = load atomic i32, ptr [[ALIGNEDADDR]] monotonic, align 4
+; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
+; CHECK: atomicrmw.start:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
+; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i16 [[EXTRACTED]] to <2 x i8>
+; CHECK-NEXT: [[NEW:%.*]] = add <2 x i8> [[TMP3]], [[VAL:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i8> [[NEW]] to i16
+; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP4]] to i32
+; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
+; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
+; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK: atomicrmw.end:
+; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
+; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i16 [[EXTRACTED3]] to <2 x i8>
+; CHECK-NEXT: ret <2 x i8> [[TMP6]]
+;
+entry:
+ %old = atomicrmw elementwise add ptr %addr, <2 x i8> %val monotonic
+ ret <2 x i8> %old
+}
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
new file mode 100644
index 0000000000000..a19b5f0e0c1cd
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=x86_64-linux-gnu | FileCheck %s
+
+; By default, elementwise atomicrmws are treated the same as the non-elementwise versions and are not expanded.
+; Targets in the future can take advantage of elementwise expansion by returning AtomicExpansionKind::Elementwise from shouldExpandAtomicRMWInIR.
+; Elementwise atomics work on integer vectors, but normal atomics don't.
+
+define <4 x i32> @atomicrmw_add_elementwise(ptr %p, <4 x i32> %v) {
+; CHECK-LABEL: define <4 x i32> @atomicrmw_add_elementwise(
+; CHECK-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[V]] to i128
+; CHECK-NEXT: [[TMP2:%.*]] = call i128 @__atomic_fetch_add_16(ptr [[P]], i128 [[TMP1]], i32 0)
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i128 [[TMP2]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[TMP3]]
+;
+ %old = atomicrmw elementwise add ptr %p, <4 x i32> %v monotonic
+ ret <4 x i32> %old
+}
+
+define <4 x float> @atomicrmw_fadd_elementwise(ptr %p, <4 x float> %v) {
+; CHECK-LABEL: define <4 x float> @atomicrmw_fadd_elementwise(
+; CHECK-SAME: ptr [[P:%.*]], <4 x float> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca <4 x float>, align 16
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[P]], align 16
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <4 x float> [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd <4 x float> [[LOADED]], [[V]]
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT: store <4 x float> [[LOADED]], ptr [[TMP1]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x float> [[NEW]] to i128
+; CHECK-NEXT: [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_16(ptr [[P]], ptr [[TMP1]], i128 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP5:%.*]] = load <4 x float>, ptr [[TMP1]], align 16
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT: [[TMP6:%.*]] = insertvalue { <4 x float>, i1 } poison, <4 x float> [[TMP5]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertvalue { <4 x float>, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { <4 x float>, i1 } [[TMP7]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { <4 x float>, i1 } [[TMP7]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret <4 x float> [[NEWLOADED]]
+;
+ %old = atomicrmw elementwise fadd ptr %p, <4 x float> %v monotonic
+ ret <4 x float> %old
+}
+
+define <4 x float> @atomicrmw_fadd(ptr %p, <4 x float> %v) {
+; CHECK-LABEL: define <4 x float> @atomicrmw_fadd(
+; CHECK-SAME: ptr [[P:%.*]], <4 x float> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca <4 x float>, align 16
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[P]], align 16
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <4 x float> [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd <4 x float> [[LOADED]], [[V]]
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT: store <4 x float> [[LOADED]], ptr [[TMP1]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x float> [[NEW]] to i128
+; CHECK-NEXT: [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_16(ptr [[P]], ptr [[TMP1]], i128 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP5:%.*]] = load <4 x float>, ptr [[TMP1]], align 16
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT: [[TMP6:%.*]] = insertvalue { <4 x float>, i1 } poison, <4 x float> [[TMP5]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertvalue { <4 x float>, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { <4 x float>, i1 } [[TMP7]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { <4 x float>, i1 } [[TMP7]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret <4 x float> [[NEWLOADED]]
+;
+ %old = atomicrmw fadd ptr %p, <4 x float> %v monotonic
+ ret <4 x float> %old
+}
>From 45f62f18d49529c4d171d1e0f89824152474a315 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 8 May 2026 18:31:08 +0000
Subject: [PATCH 2/5] format
---
llvm/include/llvm/CodeGen/TargetLowering.h | 10 +++++-----
llvm/lib/CodeGen/AtomicExpandPass.cpp | 17 +++++++++--------
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 10 +++++++---
3 files changed, 21 insertions(+), 16 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 99c43a6594bf0..d8ae110b84f85 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -274,9 +274,10 @@ class LLVM_ABI TargetLoweringBase {
// each half. Split an `atomicrmw elementwise <N x T>` into either
// two `atomicrmw elementwise <N/2 x T>` (when N > 2) or two scalar
// `atomicrmw T` (when N == 2). An `<1 x T>` elementwise RMW is collapsed
- // directly to a scalar `atomicrmw T`. Each resulting atomicrmw is fed back through the
- // expansion pipeline, so the target's `shouldExpandAtomicRMWInIR` is
- // re-queried at the halved width and may return any expansion kind there:
+ // directly to a scalar `atomicrmw T`. Each resulting atomicrmw is fed back
+ // through the expansion pipeline, so the target's
+ // `shouldExpandAtomicRMWInIR` is re-queried at the halved width and may
+ // return any expansion kind there:
// - `None` to preserve at that width (e.g. a native vector atomic),
// - `Elementwise` to keep halving,
// - `CmpXChg` (or any other kind) to commit to that expansion for the
@@ -2514,8 +2515,7 @@ class LLVM_ABI TargetLoweringBase {
return AtomicExpansionKind::CastToInteger;
if (Ty->isVectorTy()) {
- TypeSize SizeInBits =
- RMWI->getDataLayout().getTypeStoreSizeInBits(Ty);
+ TypeSize SizeInBits = RMWI->getDataLayout().getTypeStoreSizeInBits(Ty);
if (!SizeInBits.isScalable() &&
SizeInBits.getFixedValue() <= getMaxAtomicSizeInBitsSupported())
return AtomicExpansionKind::CastToInteger;
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 970d10bb1e59b..35b75932d0a2a 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -386,10 +386,10 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
auto ExpansionKind = TLI->shouldExpandAtomicRMWInIR(RMWI);
if (ExpansionKind ==
TargetLoweringBase::AtomicExpansionKind::Elementwise) {
- // If the target wants fence-based lowering for this elementwise RMW, insert
- // the fences before splitting and downgrade the RMW ordering first.
- // The split operations inherit the downgraded ordering, so they do not each
- // get their own fence pair.
+ // If the target wants fence-based lowering for this elementwise RMW,
+ // insert the fences before splitting and downgrade the RMW ordering
+ // first. The split operations inherit the downgraded ordering, so they
+ // do not each get their own fence pair.
tryInsertFencesForAtomic(
RMWI,
isReleaseOrStronger(RMWI->getOrdering()) ||
@@ -639,7 +639,8 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
/// Note that halving works because the vectors must always be a power of two.
///
/// Each new atomicrmw inherits the original ordering/syncscope/volatility and
-/// metadata, and is fed back through processAtomicInstr() so further expansion fires per-half.
+/// metadata, and is fed back through processAtomicInstr() so further expansion
+/// fires per-half.
void AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
assert(AI->isElementwise());
auto *VecTy = cast<FixedVectorType>(AI->getType());
@@ -1277,9 +1278,9 @@ AtomicRMWInst *AtomicExpandImpl::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
// or/xor/and on vectors are equivalent to the same operation on an integer
// that spans the vector, so we can use the integer type for the operation.
ValOp = Builder.CreateBitCast(ValOp, PMV.IntValueType);
- Value *ValOperand_Shifted = Builder.CreateShl(
- Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
- "ValOperand_Shifted");
+ Value *ValOperand_Shifted =
+ Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
+ "ValOperand_Shifted");
Value *NewOperand;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 59842422f0168..84ca7dd9a3830 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7481,7 +7481,10 @@ getScalarAtomicRMWExpansion(AtomicRMWInst::BinOp Op, Type *Ty,
return AtomicExpansionKind::CmpXChg;
}
- assert(Ty->isIntegerTy() && "Ty should be integer at this point. Integer operations must act on an integer operand. We already cast non-integer CmpXChg operands to integer.");
+ assert(Ty->isIntegerTy() &&
+ "Ty should be integer at this point. Integer operations must act on "
+ "an integer operand. We already cast non-integer CmpXChg operands to "
+ "integer.");
const unsigned BitWidth = cast<IntegerType>(Ty)->getBitWidth();
switch (Op) {
@@ -7684,8 +7687,9 @@ Instruction *NVPTXTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
bool IsEmulated = false;
if (CI)
- IsEmulated = cast<IntegerType>(CI->getCompareOperand()->getType())
- ->getBitWidth() < STI.getMinCmpXchgSizeInBits();
+ IsEmulated =
+ cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth() <
+ STI.getMinCmpXchgSizeInBits();
else {
AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(RI);
IsEmulated = Expansion == AtomicExpansionKind::CmpXChg ||
>From 83f6cc3bbfe0ce9db45349f6fcc968f0a7c6cdc2 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 8 May 2026 23:14:08 +0000
Subject: [PATCH 3/5] updated comment
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 28 ++++++++++++++-------
1 file changed, 19 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 84ca7dd9a3830..b1dea77ce18b2 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7585,9 +7585,11 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
const Instruction *I) const {
- // This function returns true iff the operation is emulated using a CAS-loop,
- // or if it has the memory order seq_cst (which is not natively supported in
- // the PTX `atom` instruction).
+ // This function returns true iff AtomicExpandPass should enforce the
+ // instruction's ordering with fences instead of leaving that ordering on the
+ // atomic instruction itself. This covers CAS-loop emulation, seq_cst
+ // operations (which are not natively supported by the PTX `atom`
+ // instruction), and elementwise atomicrmw expansion.
//
// atomicrmw and cmpxchg instructions not efficiently supported by PTX
// are lowered to CAS emulation loops that preserve their memory order,
@@ -7595,8 +7597,13 @@ bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
// atom.cas.relaxed.sco instructions within the loop, and fences before and
// after the loop to restore order.
//
+ // For ordered elementwise atomicrmw, if we expand elementwise, it is also
+ // more efficient to insert fences around the whole split sequence once and
+ // downgrade the ordering for the entire sequence, rather than having the
+ // stronger ordering on every expanded atomicrmw.
+ //
// Atomic instructions efficiently supported by PTX are lowered to
- // `atom.<op>.<sem>.<scope` instruction with their corresponding memory order
+ // `atom.<op>.<sem>.<scope>` instruction with their corresponding memory order
// and scope. Since PTX does not support seq_cst, we emulate it by lowering to
// a fence.sc followed by an atom according to the PTX atomics ABI
// https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
@@ -7638,7 +7645,7 @@ AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth() >=
STI.getMinCmpXchgSizeInBits())
return AtomicOrdering::Acquire;
- else if (auto *RI = dyn_cast<AtomicRMWInst>(I);
+ if (auto *RI = dyn_cast<AtomicRMWInst>(I);
RI && RI->getOrdering() == AtomicOrdering::SequentiallyConsistent &&
shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::None)
return AtomicOrdering::Acquire;
@@ -7685,18 +7692,21 @@ Instruction *NVPTXTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
auto SSID = getAtomicSyncScopeID(Inst);
assert(SSID.has_value() && "Expected an atomic operation");
- bool IsEmulated = false;
+ if (!isAcquireOrStronger(Ord))
+ return nullptr;
+
+ bool NeedsTrailingAquireFence = false;
if (CI)
- IsEmulated =
+ NeedsTrailingAquireFence =
cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth() <
STI.getMinCmpXchgSizeInBits();
else {
AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(RI);
- IsEmulated = Expansion == AtomicExpansionKind::CmpXChg ||
+ NeedsTrailingAquireFence = Expansion == AtomicExpansionKind::CmpXChg ||
Expansion == AtomicExpansionKind::Elementwise;
}
- if (isAcquireOrStronger(Ord) && IsEmulated)
+ if (NeedsTrailingAquireFence)
return Builder.CreateFence(AtomicOrdering::Acquire, SSID.value());
return nullptr;
>From 36eacb2e1c528d432cc01054d0d1e6f89991cf8e Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Sat, 9 May 2026 01:14:06 +0000
Subject: [PATCH 4/5] size 1 vectors
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 4 +
.../NVPTX/atomicrmw-elementwise-sm60.ll | 6570 ++++++++++++-----
.../NVPTX/atomicrmw-elementwise-sm70.ll | 6554 +++++++++++-----
.../NVPTX/atomicrmw-elementwise-sm90.ll | 6024 +++++++++++----
llvm/test/CodeGen/NVPTX/atomicrmw.py | 69 +
.../NVPTX/expand-atomic-rmw-elementwise.ll | 15 +
6 files changed, 14348 insertions(+), 4888 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index b1dea77ce18b2..6fc2c316ccb81 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7559,6 +7559,10 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
auto *VecTy = cast<FixedVectorType>(AI->getType());
Type *LaneTy = VecTy->getElementType();
+ // Collapse <1 x T> elementwise RMWs to scalar RMWs
+ if (VecTy->getNumElements() == 1)
+ return AtomicExpansionKind::Elementwise;
+
// If the scalar lane op is natively supported, return
// Elementwise so halving eventually bottoms out at the scalar base
// case, where this hook returns None for each scalar lane and the
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
index bb279c9a92e5f..875a7f40fab93 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
@@ -2,6 +2,43 @@
; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | FileCheck %s --check-prefix=SM60
; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | %ptxas-verify -arch=sm_60 %}
+define <1 x i8> @xchg_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: xchg_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<14>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r5, [xchg_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
+; SM60-NEXT: mov.b32 %r8, 255;
+; SM60-NEXT: shl.b32 %r9, %r8, %r1;
+; SM60-NEXT: not.b32 %r2, %r9;
+; SM60-NEXT: shl.b32 %r3, %r5, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r13, [%rd1];
+; SM60-NEXT: $L__BB0_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r13, %r2;
+; SM60-NEXT: or.b32 %r11, %r10, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM60-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM60-NEXT: mov.b32 %r13, %r4;
+; SM60-NEXT: @%p1 bra $L__BB0_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r12, %r4, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r12;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: xchg_acq_rel_v2i8_global_cta(
; SM60: {
@@ -22,14 +59,14 @@ define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: not.b32 %r2, %r9;
; SM60-NEXT: shl.b32 %r3, %r5, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r13, [%rd1];
-; SM60-NEXT: $L__BB0_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB1_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r10, %r13, %r2;
; SM60-NEXT: or.b32 %r11, %r10, %r3;
; SM60-NEXT: atom.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
; SM60-NEXT: setp.ne.b32 %p1, %r4, %r13;
; SM60-NEXT: mov.b32 %r13, %r4;
-; SM60-NEXT: @%p1 bra $L__BB0_1;
+; SM60-NEXT: @%p1 bra $L__BB1_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r12, %r4, %r1;
; SM60-NEXT: membar.cta;
@@ -70,6 +107,43 @@ define <8 x i8> @xchg_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
ret <8 x i8> %retval
}
+define <1 x i16> @xchg_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: xchg_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<14>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
+; SM60-NEXT: mov.b32 %r8, 65535;
+; SM60-NEXT: shl.b32 %r9, %r8, %r1;
+; SM60-NEXT: not.b32 %r2, %r9;
+; SM60-NEXT: shl.b32 %r3, %r5, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r13, [%rd1];
+; SM60-NEXT: $L__BB4_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r13, %r2;
+; SM60-NEXT: or.b32 %r11, %r10, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM60-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM60-NEXT: mov.b32 %r13, %r4;
+; SM60-NEXT: @%p1 bra $L__BB4_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r12, %r4, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r12;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @xchg_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: xchg_acq_rel_v2i16_global_cta(
; SM60: {
@@ -128,6 +202,24 @@ define <8 x i16> @xchg_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
ret <8 x i16> %retval
}
+define <1 x i32> @xchg_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: xchg_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @xchg_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: xchg_acq_rel_v2i32_global_cta(
; SM60: {
@@ -195,6 +287,23 @@ define <8 x i32> @xchg_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i64> @xchg_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: xchg_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @xchg_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: xchg_acq_rel_v2i64_global_cta(
; SM60: {
@@ -265,6 +374,45 @@ define <8 x i64> @xchg_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
ret <8 x i64> %retval
}
+define <1 x i8> @add_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: add_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [add_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB16_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB16_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: add_acq_rel_v2i8_global_cta(
; SM60: {
@@ -287,7 +435,7 @@ define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM60-NEXT: $L__BB12_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB17_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r15, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -304,7 +452,7 @@ define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
; SM60-NEXT: mov.b32 %r15, %r3;
-; SM60-NEXT: @%p1 bra $L__BB12_1;
+; SM60-NEXT: @%p1 bra $L__BB17_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r14, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -335,7 +483,7 @@ define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: $L__BB13_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB18_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -359,7 +507,7 @@ define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM60-NEXT: mov.b32 %r18, %r1;
-; SM60-NEXT: @%p1 bra $L__BB13_1;
+; SM60-NEXT: @%p1 bra $L__BB18_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -393,7 +541,7 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM60-NEXT: $L__BB14_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB19_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -451,7 +599,7 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB14_1;
+; SM60-NEXT: @%p1 bra $L__BB19_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -460,6 +608,45 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @add_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: add_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r6, [add_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 65535;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB20_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB20_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: add_acq_rel_v2i16_global_cta(
; SM60: {
@@ -474,7 +661,7 @@ define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM60-NEXT: $L__BB15_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB21_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM60-NEXT: add.s16 %rs5, %rs4, %rs2;
@@ -483,7 +670,7 @@ define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB15_1;
+; SM60-NEXT: @%p1 bra $L__BB21_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -509,7 +696,7 @@ define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: $L__BB16_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB22_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
; SM60-NEXT: add.s16 %rs5, %rs4, %rs2;
@@ -531,7 +718,7 @@ define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB16_1;
+; SM60-NEXT: @%p1 bra $L__BB22_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -557,7 +744,7 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: $L__BB17_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB23_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
; SM60-NEXT: add.s16 %rs5, %rs4, %rs2;
@@ -579,14 +766,14 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB17_1;
+; SM60-NEXT: @%p1 bra $L__BB23_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM60-NEXT: $L__BB17_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB23_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
; SM60-NEXT: add.s16 %rs17, %rs16, %rs14;
@@ -608,7 +795,7 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB17_3;
+; SM60-NEXT: @%p2 bra $L__BB23_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -617,6 +804,24 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @add_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: add_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [add_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @add_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: add_acq_rel_v2i32_global_cta(
; SM60: {
@@ -684,6 +889,23 @@ define <8 x i32> @add_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @add_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: add_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @add_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: add_acq_rel_v2i64_global_cta(
; SM60: {
@@ -754,6 +976,45 @@ define <8 x i64> @add_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @sub_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: sub_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [sub_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB32_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB32_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: sub_acq_rel_v2i8_global_cta(
; SM60: {
@@ -776,7 +1037,7 @@ define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM60-NEXT: $L__BB24_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB33_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r15, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -793,7 +1054,7 @@ define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
; SM60-NEXT: mov.b32 %r15, %r3;
-; SM60-NEXT: @%p1 bra $L__BB24_1;
+; SM60-NEXT: @%p1 bra $L__BB33_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r14, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -824,7 +1085,7 @@ define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: $L__BB25_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB34_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -848,7 +1109,7 @@ define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM60-NEXT: mov.b32 %r18, %r1;
-; SM60-NEXT: @%p1 bra $L__BB25_1;
+; SM60-NEXT: @%p1 bra $L__BB34_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -882,7 +1143,7 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM60-NEXT: $L__BB26_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB35_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -940,7 +1201,7 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB26_1;
+; SM60-NEXT: @%p1 bra $L__BB35_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -949,6 +1210,45 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @sub_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: sub_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r6, [sub_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 65535;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB36_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB36_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: sub_acq_rel_v2i16_global_cta(
; SM60: {
@@ -963,7 +1263,7 @@ define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM60-NEXT: $L__BB27_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB37_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -972,7 +1272,7 @@ define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB27_1;
+; SM60-NEXT: @%p1 bra $L__BB37_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -998,7 +1298,7 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: $L__BB28_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB38_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -1020,7 +1320,7 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB28_1;
+; SM60-NEXT: @%p1 bra $L__BB38_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -1046,7 +1346,7 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: $L__BB29_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB39_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -1068,14 +1368,14 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB29_1;
+; SM60-NEXT: @%p1 bra $L__BB39_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM60-NEXT: $L__BB29_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB39_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
; SM60-NEXT: sub.s16 %rs17, %rs16, %rs14;
@@ -1097,7 +1397,7 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB29_3;
+; SM60-NEXT: @%p2 bra $L__BB39_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -1106,6 +1406,25 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @sub_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: sub_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<4>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [sub_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: neg.s32 %r2, %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: sub_acq_rel_v2i32_global_cta(
; SM60: {
@@ -1187,6 +1506,24 @@ define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @sub_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: sub_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<5>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: neg.s64 %rd3, %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd4, [%rd1], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd4;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: sub_acq_rel_v2i64_global_cta(
; SM60: {
@@ -1271,6 +1608,34 @@ define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @and_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: and_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<12>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r1, [and_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM60-NEXT: and.b32 %r3, %r2, 3;
+; SM60-NEXT: shl.b32 %r4, %r3, 3;
+; SM60-NEXT: mov.b32 %r5, 255;
+; SM60-NEXT: shl.b32 %r6, %r5, %r4;
+; SM60-NEXT: not.b32 %r7, %r6;
+; SM60-NEXT: shl.b32 %r8, %r1, %r4;
+; SM60-NEXT: or.b32 %r9, %r8, %r7;
+; SM60-NEXT: atom.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM60-NEXT: shr.u32 %r11, %r10, %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r11;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @and_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: and_acq_rel_v2i8_global_cta(
; SM60: {
@@ -1311,13 +1676,13 @@ define <4 x i8> @and_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NEXT: $L__BB37_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB50_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r3, %r4, %r2;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB37_1;
+; SM60-NEXT: @%p1 bra $L__BB50_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -1340,7 +1705,7 @@ define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB38_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB51_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r3, %r5, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -1356,7 +1721,7 @@ define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB38_1;
+; SM60-NEXT: @%p1 bra $L__BB51_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -1365,6 +1730,34 @@ define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @and_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: and_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<12>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r1, [and_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM60-NEXT: and.b32 %r3, %r2, 3;
+; SM60-NEXT: shl.b32 %r4, %r3, 3;
+; SM60-NEXT: mov.b32 %r5, 65535;
+; SM60-NEXT: shl.b32 %r6, %r5, %r4;
+; SM60-NEXT: not.b32 %r7, %r6;
+; SM60-NEXT: shl.b32 %r8, %r1, %r4;
+; SM60-NEXT: or.b32 %r9, %r8, %r7;
+; SM60-NEXT: atom.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM60-NEXT: shr.u32 %r11, %r10, %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r11;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: and_acq_rel_v2i16_global_cta(
; SM60: {
@@ -1377,13 +1770,13 @@ define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i16_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NEXT: $L__BB39_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB53_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r3, %r4, %r2;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB39_1;
+; SM60-NEXT: @%p1 bra $L__BB53_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -1406,7 +1799,7 @@ define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB40_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB54_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r3, %r5, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -1422,7 +1815,7 @@ define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB40_1;
+; SM60-NEXT: @%p1 bra $L__BB54_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -1445,7 +1838,7 @@ define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB41_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB55_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r5, %r9, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -1461,12 +1854,12 @@ define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB41_1;
+; SM60-NEXT: @%p1 bra $L__BB55_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: $L__BB41_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB55_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r7, %r11, %r3;
; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -1482,7 +1875,7 @@ define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB41_3;
+; SM60-NEXT: @%p2 bra $L__BB55_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -1491,18 +1884,36 @@ define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
-define <2 x i32> @and_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM60-LABEL: and_acq_rel_v2i32_global_cta(
+define <1 x i32> @and_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: and_acq_rel_v1i32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b32 %r<3>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i32_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v2i32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.and.b32 %r3, [%rd1], %r1;
-; SM60-NEXT: atom.cta.global.and.b32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: ld.param.b32 %r1, [and_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @and_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: and_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.and.b32 %r4, [%rd1+4], %r2;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM60-NEXT: ret;
@@ -1558,6 +1969,23 @@ define <8 x i32> @and_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @and_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: and_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [and_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @and_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: and_acq_rel_v2i64_global_cta(
; SM60: {
@@ -1628,6 +2056,46 @@ define <8 x i64> @and_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @nand_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: nand_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [nand_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
+; SM60-NEXT: @%p1 bra $L__BB64_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: nand_acq_rel_v2i8_global_cta(
; SM60: {
@@ -1649,7 +2117,7 @@ define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: shl.b32 %r8, %r7, %r1;
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
-; SM60-NEXT: $L__BB48_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB65_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r18, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -1667,7 +2135,7 @@ define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM60-NEXT: mov.b32 %r18, %r3;
-; SM60-NEXT: @%p1 bra $L__BB48_1;
+; SM60-NEXT: @%p1 bra $L__BB65_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r17, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -1689,14 +2157,14 @@ define <4 x i8> @nand_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB49_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB66_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r3, %r5, %r2;
; SM60-NEXT: xor.b32 %r4, %r3, -1;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM60-NEXT: mov.b32 %r5, %r1;
-; SM60-NEXT: @%p1 bra $L__BB49_1;
+; SM60-NEXT: @%p1 bra $L__BB66_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -1719,7 +2187,7 @@ define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: $L__BB50_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB67_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -1737,7 +2205,7 @@ define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB50_1;
+; SM60-NEXT: @%p1 bra $L__BB67_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -1746,6 +2214,46 @@ define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
ret <8 x i8> %retval
}
+define <1 x i16> @nand_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: nand_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r6, [nand_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 65535;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: $L__BB68_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
+; SM60-NEXT: @%p1 bra $L__BB68_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: nand_acq_rel_v2i16_global_cta(
; SM60: {
@@ -1758,14 +2266,14 @@ define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i16_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB51_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB69_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r3, %r5, %r2;
; SM60-NEXT: xor.b32 %r4, %r3, -1;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM60-NEXT: mov.b32 %r5, %r1;
-; SM60-NEXT: @%p1 bra $L__BB51_1;
+; SM60-NEXT: @%p1 bra $L__BB69_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -1788,7 +2296,7 @@ define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: $L__BB52_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB70_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -1806,7 +2314,7 @@ define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB52_1;
+; SM60-NEXT: @%p1 bra $L__BB70_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -1829,7 +2337,7 @@ define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
-; SM60-NEXT: $L__BB53_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB71_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
; SM60-NEXT: cvt.u64.u32 %rd4, %r14;
@@ -1847,12 +2355,12 @@ define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB53_1;
+; SM60-NEXT: @%p1 bra $L__BB71_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r15, %rd12;
-; SM60-NEXT: $L__BB53_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB71_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd13, %r15;
; SM60-NEXT: cvt.u64.u32 %rd14, %r16;
@@ -1870,7 +2378,7 @@ define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r15, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB53_3;
+; SM60-NEXT: @%p2 bra $L__BB71_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
@@ -1879,6 +2387,34 @@ define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
ret <8 x i16> %retval
}
+define <1 x i32> @nand_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: nand_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [nand_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: not.b32 %r4, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p1 bra $L__BB72_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: nand_acq_rel_v2i32_global_cta(
; SM60: {
@@ -1893,7 +2429,7 @@ define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB54_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB73_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -1910,7 +2446,7 @@ define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB54_1;
+; SM60-NEXT: @%p1 bra $L__BB73_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -1933,7 +2469,7 @@ define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB55_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB74_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -1950,12 +2486,12 @@ define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB55_1;
+; SM60-NEXT: @%p1 bra $L__BB74_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM60-NEXT: $L__BB55_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB74_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
@@ -1972,7 +2508,7 @@ define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB55_3;
+; SM60-NEXT: @%p2 bra $L__BB74_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -1996,7 +2532,7 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM60-NEXT: $L__BB56_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB75_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -2013,12 +2549,12 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB56_1;
+; SM60-NEXT: @%p1 bra $L__BB75_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM60-NEXT: $L__BB56_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB75_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -2035,12 +2571,12 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB56_3;
+; SM60-NEXT: @%p2 bra $L__BB75_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM60-NEXT: $L__BB56_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB75_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -2057,12 +2593,12 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM60-NEXT: @%p3 bra $L__BB56_5;
+; SM60-NEXT: @%p3 bra $L__BB75_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM60-NEXT: $L__BB56_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB75_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -2079,7 +2615,7 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM60-NEXT: @%p4 bra $L__BB56_7;
+; SM60-NEXT: @%p4 bra $L__BB75_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -2089,6 +2625,33 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i64> @nand_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: nand_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b64 %rd<7>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd3];
+; SM60-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd4, %rd6, %rd1;
+; SM60-NEXT: not.b64 %rd5, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
+; SM60-NEXT: mov.b64 %rd6, %rd2;
+; SM60-NEXT: @%p1 bra $L__BB76_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: nand_acq_rel_v2i64_global_cta(
; SM60: {
@@ -2100,24 +2663,24 @@ define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v2i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
-; SM60-NEXT: $L__BB57_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB77_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b64 %rd6, %rd10, %rd1;
; SM60-NEXT: not.b64 %rd7, %rd6;
; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
; SM60-NEXT: mov.b64 %rd10, %rd3;
-; SM60-NEXT: @%p1 bra $L__BB57_1;
+; SM60-NEXT: @%p1 bra $L__BB77_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
-; SM60-NEXT: $L__BB57_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB77_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b64 %rd8, %rd11, %rd2;
; SM60-NEXT: not.b64 %rd9, %rd8;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
; SM60-NEXT: mov.b64 %rd11, %rd4;
-; SM60-NEXT: @%p2 bra $L__BB57_3;
+; SM60-NEXT: @%p2 bra $L__BB77_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -2138,44 +2701,44 @@ define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v4i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
-; SM60-NEXT: $L__BB58_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB78_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd5, %rd18;
; SM60-NEXT: and.b64 %rd10, %rd5, %rd3;
; SM60-NEXT: not.b64 %rd11, %rd10;
; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
; SM60-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
-; SM60-NEXT: @%p1 bra $L__BB58_1;
+; SM60-NEXT: @%p1 bra $L__BB78_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
-; SM60-NEXT: $L__BB58_3: // %atomicrmw.start5
+; SM60-NEXT: $L__BB78_3: // %atomicrmw.start5
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd6, %rd19;
; SM60-NEXT: and.b64 %rd12, %rd6, %rd4;
; SM60-NEXT: not.b64 %rd13, %rd12;
; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
; SM60-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
-; SM60-NEXT: @%p2 bra $L__BB58_3;
+; SM60-NEXT: @%p2 bra $L__BB78_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end4
; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
-; SM60-NEXT: $L__BB58_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB78_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b64 %rd14, %rd20, %rd1;
; SM60-NEXT: not.b64 %rd15, %rd14;
; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
; SM60-NEXT: mov.b64 %rd20, %rd7;
-; SM60-NEXT: @%p3 bra $L__BB58_5;
+; SM60-NEXT: @%p3 bra $L__BB78_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
-; SM60-NEXT: $L__BB58_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB78_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b64 %rd16, %rd21, %rd2;
; SM60-NEXT: not.b64 %rd17, %rd16;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
; SM60-NEXT: mov.b64 %rd21, %rd8;
-; SM60-NEXT: @%p4 bra $L__BB58_7;
+; SM60-NEXT: @%p4 bra $L__BB78_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -2199,84 +2762,84 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v8i64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [nand_acq_rel_v8i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
-; SM60-NEXT: $L__BB59_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB79_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd7, %rd34;
; SM60-NEXT: and.b64 %rd18, %rd7, %rd5;
; SM60-NEXT: not.b64 %rd19, %rd18;
; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
; SM60-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
-; SM60-NEXT: @%p1 bra $L__BB59_1;
+; SM60-NEXT: @%p1 bra $L__BB79_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
-; SM60-NEXT: $L__BB59_3: // %atomicrmw.start9
+; SM60-NEXT: $L__BB79_3: // %atomicrmw.start9
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd8, %rd35;
; SM60-NEXT: and.b64 %rd20, %rd8, %rd6;
; SM60-NEXT: not.b64 %rd21, %rd20;
; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
; SM60-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
-; SM60-NEXT: @%p2 bra $L__BB59_3;
+; SM60-NEXT: @%p2 bra $L__BB79_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end8
; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
-; SM60-NEXT: $L__BB59_5: // %atomicrmw.start20
+; SM60-NEXT: $L__BB79_5: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd9, %rd36;
; SM60-NEXT: and.b64 %rd22, %rd9, %rd3;
; SM60-NEXT: not.b64 %rd23, %rd22;
; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
; SM60-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
-; SM60-NEXT: @%p3 bra $L__BB59_5;
+; SM60-NEXT: @%p3 bra $L__BB79_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end19
; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
-; SM60-NEXT: $L__BB59_7: // %atomicrmw.start26
+; SM60-NEXT: $L__BB79_7: // %atomicrmw.start26
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd10, %rd37;
; SM60-NEXT: and.b64 %rd24, %rd10, %rd4;
; SM60-NEXT: not.b64 %rd25, %rd24;
; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
; SM60-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
-; SM60-NEXT: @%p4 bra $L__BB59_7;
+; SM60-NEXT: @%p4 bra $L__BB79_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end25
; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
-; SM60-NEXT: $L__BB59_9: // %atomicrmw.start41
+; SM60-NEXT: $L__BB79_9: // %atomicrmw.start41
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd13, %rd38;
; SM60-NEXT: and.b64 %rd26, %rd13, %rd1;
; SM60-NEXT: not.b64 %rd27, %rd26;
; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
; SM60-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
-; SM60-NEXT: @%p5 bra $L__BB59_9;
+; SM60-NEXT: @%p5 bra $L__BB79_9;
; SM60-NEXT: // %bb.10: // %atomicrmw.end40
; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
-; SM60-NEXT: $L__BB59_11: // %atomicrmw.start47
+; SM60-NEXT: $L__BB79_11: // %atomicrmw.start47
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd14, %rd39;
; SM60-NEXT: and.b64 %rd28, %rd14, %rd2;
; SM60-NEXT: not.b64 %rd29, %rd28;
; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
; SM60-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
-; SM60-NEXT: @%p6 bra $L__BB59_11;
+; SM60-NEXT: @%p6 bra $L__BB79_11;
; SM60-NEXT: // %bb.12: // %atomicrmw.end46
; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
-; SM60-NEXT: $L__BB59_13: // %atomicrmw.start58
+; SM60-NEXT: $L__BB79_13: // %atomicrmw.start58
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b64 %rd30, %rd40, %rd11;
; SM60-NEXT: not.b64 %rd31, %rd30;
; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
; SM60-NEXT: mov.b64 %rd40, %rd15;
-; SM60-NEXT: @%p7 bra $L__BB59_13;
+; SM60-NEXT: @%p7 bra $L__BB79_13;
; SM60-NEXT: // %bb.14: // %atomicrmw.end57
; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
-; SM60-NEXT: $L__BB59_15: // %atomicrmw.start64
+; SM60-NEXT: $L__BB79_15: // %atomicrmw.start64
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b64 %rd32, %rd41, %rd12;
; SM60-NEXT: not.b64 %rd33, %rd32;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
; SM60-NEXT: mov.b64 %rd41, %rd16;
-; SM60-NEXT: @%p8 bra $L__BB59_15;
+; SM60-NEXT: @%p8 bra $L__BB79_15;
; SM60-NEXT: // %bb.16: // %atomicrmw.end63
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -2288,6 +2851,30 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
ret <8 x i64> %retval
}
+define <1 x i8> @or_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: or_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r1, [or_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM60-NEXT: and.b32 %r3, %r2, 3;
+; SM60-NEXT: shl.b32 %r4, %r3, 3;
+; SM60-NEXT: shl.b32 %r5, %r1, %r4;
+; SM60-NEXT: atom.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM60-NEXT: shr.u32 %r7, %r6, %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r7;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @or_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: or_acq_rel_v2i8_global_cta(
; SM60: {
@@ -2324,13 +2911,13 @@ define <4 x i8> @or_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %va
; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB82_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: or.b32 %r3, %r4, %r2;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB61_1;
+; SM60-NEXT: @%p1 bra $L__BB82_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -2353,7 +2940,7 @@ define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %va
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB83_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: or.b32 %r3, %r5, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -2369,7 +2956,7 @@ define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %va
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB62_1;
+; SM60-NEXT: @%p1 bra $L__BB83_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -2378,6 +2965,30 @@ define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %va
ret <8 x i8> %retval
}
+define <1 x i16> @or_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: or_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r1, [or_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM60-NEXT: and.b32 %r3, %r2, 3;
+; SM60-NEXT: shl.b32 %r4, %r3, 3;
+; SM60-NEXT: shl.b32 %r5, %r1, %r4;
+; SM60-NEXT: atom.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM60-NEXT: shr.u32 %r7, %r6, %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r7;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: or_acq_rel_v2i16_global_cta(
; SM60: {
@@ -2390,13 +3001,13 @@ define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i16_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB85_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: or.b32 %r3, %r4, %r2;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB63_1;
+; SM60-NEXT: @%p1 bra $L__BB85_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -2419,7 +3030,7 @@ define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB86_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: or.b32 %r3, %r5, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -2435,7 +3046,7 @@ define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB64_1;
+; SM60-NEXT: @%p1 bra $L__BB86_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -2458,7 +3069,7 @@ define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB87_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: or.b32 %r5, %r9, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -2474,12 +3085,12 @@ define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB65_1;
+; SM60-NEXT: @%p1 bra $L__BB87_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: $L__BB65_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB87_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: or.b32 %r7, %r11, %r3;
; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -2495,7 +3106,7 @@ define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB65_3;
+; SM60-NEXT: @%p2 bra $L__BB87_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -2504,6 +3115,24 @@ define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @or_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: or_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [or_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @or_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: or_acq_rel_v2i32_global_cta(
; SM60: {
@@ -2571,6 +3200,23 @@ define <8 x i32> @or_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @or_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: or_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [or_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @or_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: or_acq_rel_v2i64_global_cta(
; SM60: {
@@ -2641,6 +3287,30 @@ define <8 x i64> @or_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @xor_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: xor_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r1, [xor_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM60-NEXT: and.b32 %r3, %r2, 3;
+; SM60-NEXT: shl.b32 %r4, %r3, 3;
+; SM60-NEXT: shl.b32 %r5, %r1, %r4;
+; SM60-NEXT: atom.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM60-NEXT: shr.u32 %r7, %r6, %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r7;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @xor_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: xor_acq_rel_v2i8_global_cta(
; SM60: {
@@ -2677,13 +3347,13 @@ define <4 x i8> @xor_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB98_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: xor.b32 %r3, %r4, %r2;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB73_1;
+; SM60-NEXT: @%p1 bra $L__BB98_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -2706,7 +3376,7 @@ define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB99_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: xor.b32 %r3, %r5, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -2722,7 +3392,7 @@ define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB74_1;
+; SM60-NEXT: @%p1 bra $L__BB99_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -2731,6 +3401,30 @@ define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @xor_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: xor_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r1, [xor_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM60-NEXT: and.b32 %r3, %r2, 3;
+; SM60-NEXT: shl.b32 %r4, %r3, 3;
+; SM60-NEXT: shl.b32 %r5, %r1, %r4;
+; SM60-NEXT: atom.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM60-NEXT: shr.u32 %r7, %r6, %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r7;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: xor_acq_rel_v2i16_global_cta(
; SM60: {
@@ -2743,13 +3437,13 @@ define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i16_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB101_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: xor.b32 %r3, %r4, %r2;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB75_1;
+; SM60-NEXT: @%p1 bra $L__BB101_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -2772,7 +3466,7 @@ define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB102_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: xor.b32 %r3, %r5, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -2788,7 +3482,7 @@ define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB76_1;
+; SM60-NEXT: @%p1 bra $L__BB102_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -2811,7 +3505,7 @@ define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB103_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: xor.b32 %r5, %r9, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -2827,12 +3521,12 @@ define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB77_1;
+; SM60-NEXT: @%p1 bra $L__BB103_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: $L__BB77_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB103_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: xor.b32 %r7, %r11, %r3;
; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -2848,7 +3542,7 @@ define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB77_3;
+; SM60-NEXT: @%p2 bra $L__BB103_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -2857,6 +3551,24 @@ define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @xor_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: xor_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [xor_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @xor_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: xor_acq_rel_v2i32_global_cta(
; SM60: {
@@ -2924,6 +3636,23 @@ define <8 x i32> @xor_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @xor_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: xor_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [xor_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @xor_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: xor_acq_rel_v2i64_global_cta(
; SM60: {
@@ -2994,34 +3723,77 @@ define <8 x i64> @xor_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
-define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM60-LABEL: max_acq_rel_v2i8_global_cta(
+define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: max_acq_rel_v1i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<14>;
-; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<16>;
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [max_acq_rel_v2i8_global_cta_param_1];
-; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM60-NEXT: ld.param.b64 %rd2, [max_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.s8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: and.b32 %r6, %r5, 3;
-; SM60-NEXT: shl.b32 %r1, %r6, 3;
-; SM60-NEXT: mov.b32 %r7, 65535;
-; SM60-NEXT: shl.b32 %r8, %r7, %r1;
-; SM60-NEXT: not.b32 %r2, %r8;
-; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM60-NEXT: cvt.s16.s8 %rs7, %rs5;
-; SM60-NEXT: cvt.s16.s8 %rs8, %rs6;
-; SM60-NEXT: $L__BB84_1: // %atomicrmw.start
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB112_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r9, %r16, %r1;
-; SM60-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM60-NEXT: shr.u32 %r8, %r15, %r1;
+; SM60-NEXT: cvt.s8.s32 %rs2, %r8;
+; SM60-NEXT: max.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: and.b32 %r10, %r9, 255;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB112_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: max_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<14>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [max_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [max_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: cvt.s16.s8 %rs7, %rs5;
+; SM60-NEXT: cvt.s16.s8 %rs8, %rs6;
+; SM60-NEXT: $L__BB113_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r16, %r1;
+; SM60-NEXT: cvt.s8.s32 %rs3, %r9;
; SM60-NEXT: bfe.s32 %r10, %r9, 8, 8;
; SM60-NEXT: cvt.s8.s32 %rs4, %r10;
; SM60-NEXT: max.s16 %rs9, %rs4, %rs8;
@@ -3036,7 +3808,7 @@ define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r16;
; SM60-NEXT: mov.b32 %r16, %r3;
-; SM60-NEXT: @%p1 bra $L__BB84_1;
+; SM60-NEXT: @%p1 bra $L__BB113_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r15, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -3062,7 +3834,7 @@ define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
-; SM60-NEXT: $L__BB85_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB114_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
; SM60-NEXT: setp.gt.s32 %p1, %r4, %r3;
@@ -3090,7 +3862,7 @@ define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM60-NEXT: setp.ne.b32 %p5, %r1, %r26;
; SM60-NEXT: mov.b32 %r26, %r1;
-; SM60-NEXT: @%p5 bra $L__BB85_1;
+; SM60-NEXT: @%p5 bra $L__BB114_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -3116,7 +3888,7 @@ define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
-; SM60-NEXT: $L__BB86_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB115_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r49;
; SM60-NEXT: cvt.u64.u32 %rd4, %r50;
@@ -3206,7 +3978,7 @@ define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB86_1;
+; SM60-NEXT: @%p9 bra $L__BB115_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
@@ -3215,6 +3987,48 @@ define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @max_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: max_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB116_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: max.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB116_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: max_acq_rel_v2i16_global_cta(
; SM60: {
@@ -3229,7 +4043,7 @@ define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM60-NEXT: $L__BB87_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB117_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM60-NEXT: max.s16 %rs5, %rs4, %rs2;
@@ -3238,7 +4052,7 @@ define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB87_1;
+; SM60-NEXT: @%p1 bra $L__BB117_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -3264,7 +4078,7 @@ define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: $L__BB88_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB118_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -3286,7 +4100,7 @@ define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB88_1;
+; SM60-NEXT: @%p1 bra $L__BB118_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -3312,7 +4126,7 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: $L__BB89_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB119_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -3334,14 +4148,14 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB89_1;
+; SM60-NEXT: @%p1 bra $L__BB119_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM60-NEXT: $L__BB89_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB119_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -3363,7 +4177,7 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB89_3;
+; SM60-NEXT: @%p2 bra $L__BB119_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -3372,6 +4186,24 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @max_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: max_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [max_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @max_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: max_acq_rel_v2i32_global_cta(
; SM60: {
@@ -3439,6 +4271,23 @@ define <8 x i32> @max_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @max_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: max_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @max_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: max_acq_rel_v2i64_global_cta(
; SM60: {
@@ -3509,6 +4358,49 @@ define <8 x i64> @max_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: min_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.s8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB128_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r15, %r1;
+; SM60-NEXT: cvt.s8.s32 %rs2, %r8;
+; SM60-NEXT: min.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: and.b32 %r10, %r9, 255;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB128_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: min_acq_rel_v2i8_global_cta(
; SM60: {
@@ -3533,7 +4425,7 @@ define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
; SM60-NEXT: cvt.s16.s8 %rs7, %rs5;
; SM60-NEXT: cvt.s16.s8 %rs8, %rs6;
-; SM60-NEXT: $L__BB96_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB129_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r16, %r1;
; SM60-NEXT: cvt.s8.s32 %rs3, %r9;
@@ -3551,7 +4443,7 @@ define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r16;
; SM60-NEXT: mov.b32 %r16, %r3;
-; SM60-NEXT: @%p1 bra $L__BB96_1;
+; SM60-NEXT: @%p1 bra $L__BB129_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r15, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -3577,7 +4469,7 @@ define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
-; SM60-NEXT: $L__BB97_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB130_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
; SM60-NEXT: setp.le.s32 %p1, %r4, %r3;
@@ -3605,7 +4497,7 @@ define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM60-NEXT: setp.ne.b32 %p5, %r1, %r26;
; SM60-NEXT: mov.b32 %r26, %r1;
-; SM60-NEXT: @%p5 bra $L__BB97_1;
+; SM60-NEXT: @%p5 bra $L__BB130_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -3631,7 +4523,7 @@ define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
-; SM60-NEXT: $L__BB98_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB131_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r49;
; SM60-NEXT: cvt.u64.u32 %rd4, %r50;
@@ -3721,7 +4613,7 @@ define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB98_1;
+; SM60-NEXT: @%p9 bra $L__BB131_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
@@ -3730,6 +4622,48 @@ define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @min_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: min_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB132_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: min.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB132_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: min_acq_rel_v2i16_global_cta(
; SM60: {
@@ -3744,7 +4678,7 @@ define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM60-NEXT: $L__BB99_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB133_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM60-NEXT: min.s16 %rs5, %rs4, %rs2;
@@ -3753,7 +4687,7 @@ define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB99_1;
+; SM60-NEXT: @%p1 bra $L__BB133_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -3779,7 +4713,7 @@ define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: $L__BB100_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB134_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -3801,7 +4735,7 @@ define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB100_1;
+; SM60-NEXT: @%p1 bra $L__BB134_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -3827,7 +4761,7 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: $L__BB101_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB135_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -3849,14 +4783,14 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB101_1;
+; SM60-NEXT: @%p1 bra $L__BB135_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM60-NEXT: $L__BB101_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB135_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -3878,7 +4812,7 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB101_3;
+; SM60-NEXT: @%p2 bra $L__BB135_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -3887,6 +4821,24 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @min_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: min_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [min_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @min_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: min_acq_rel_v2i32_global_cta(
; SM60: {
@@ -3954,6 +4906,23 @@ define <8 x i32> @min_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @min_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: min_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @min_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: min_acq_rel_v2i64_global_cta(
; SM60: {
@@ -4024,6 +4993,49 @@ define <8 x i64> @min_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @umax_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: umax_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [umax_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB144_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: and.b16 %rs3, %rs2, 255;
+; SM60-NEXT: max.u16 %rs4, %rs3, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB144_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: umax_acq_rel_v2i8_global_cta(
; SM60: {
@@ -4046,7 +5058,7 @@ define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r4;
-; SM60-NEXT: $L__BB108_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB145_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r17, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -4065,7 +5077,7 @@ define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17;
; SM60-NEXT: mov.b32 %r17, %r3;
-; SM60-NEXT: @%p1 bra $L__BB108_1;
+; SM60-NEXT: @%p1 bra $L__BB145_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r16, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -4091,7 +5103,7 @@ define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
-; SM60-NEXT: $L__BB109_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB146_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
; SM60-NEXT: setp.gt.u32 %p1, %r4, %r3;
@@ -4111,7 +5123,7 @@ define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM60-NEXT: setp.ne.b32 %p5, %r1, %r18;
; SM60-NEXT: mov.b32 %r18, %r1;
-; SM60-NEXT: @%p5 bra $L__BB109_1;
+; SM60-NEXT: @%p5 bra $L__BB146_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -4139,7 +5151,7 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
-; SM60-NEXT: $L__BB110_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB147_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -4211,7 +5223,7 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB110_1;
+; SM60-NEXT: @%p9 bra $L__BB147_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -4220,6 +5232,48 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
ret <8 x i8> %retval
}
+define <1 x i16> @umax_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: umax_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB148_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB148_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: umax_acq_rel_v2i16_global_cta(
; SM60: {
@@ -4234,7 +5288,7 @@ define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM60-NEXT: $L__BB111_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB149_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM60-NEXT: max.u16 %rs5, %rs4, %rs2;
@@ -4243,7 +5297,7 @@ define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB111_1;
+; SM60-NEXT: @%p1 bra $L__BB149_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -4269,7 +5323,7 @@ define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: $L__BB112_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB150_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -4291,7 +5345,7 @@ define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB112_1;
+; SM60-NEXT: @%p1 bra $L__BB150_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -4317,7 +5371,7 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: $L__BB113_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB151_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -4339,14 +5393,14 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB113_1;
+; SM60-NEXT: @%p1 bra $L__BB151_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM60-NEXT: $L__BB113_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB151_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -4368,7 +5422,7 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB113_3;
+; SM60-NEXT: @%p2 bra $L__BB151_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -4377,6 +5431,24 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
ret <8 x i16> %retval
}
+define <1 x i32> @umax_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: umax_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [umax_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @umax_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: umax_acq_rel_v2i32_global_cta(
; SM60: {
@@ -4444,6 +5516,23 @@ define <8 x i32> @umax_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i64> @umax_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: umax_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @umax_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: umax_acq_rel_v2i64_global_cta(
; SM60: {
@@ -4514,6 +5603,49 @@ define <8 x i64> @umax_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
ret <8 x i64> %retval
}
+define <1 x i8> @umin_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: umin_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [umin_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB160_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: and.b16 %rs3, %rs2, 255;
+; SM60-NEXT: min.u16 %rs4, %rs3, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB160_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: umin_acq_rel_v2i8_global_cta(
; SM60: {
@@ -4536,7 +5668,7 @@ define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r4;
-; SM60-NEXT: $L__BB120_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB161_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r17, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -4555,7 +5687,7 @@ define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17;
; SM60-NEXT: mov.b32 %r17, %r3;
-; SM60-NEXT: @%p1 bra $L__BB120_1;
+; SM60-NEXT: @%p1 bra $L__BB161_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r16, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -4581,7 +5713,7 @@ define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
-; SM60-NEXT: $L__BB121_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB162_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
; SM60-NEXT: setp.le.u32 %p1, %r4, %r3;
@@ -4601,7 +5733,7 @@ define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM60-NEXT: setp.ne.b32 %p5, %r1, %r18;
; SM60-NEXT: mov.b32 %r18, %r1;
-; SM60-NEXT: @%p5 bra $L__BB121_1;
+; SM60-NEXT: @%p5 bra $L__BB162_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -4629,7 +5761,7 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
-; SM60-NEXT: $L__BB122_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB163_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -4701,7 +5833,7 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB122_1;
+; SM60-NEXT: @%p9 bra $L__BB163_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -4710,6 +5842,48 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
ret <8 x i8> %retval
}
+define <1 x i16> @umin_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: umin_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB164_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: min.u16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB164_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: umin_acq_rel_v2i16_global_cta(
; SM60: {
@@ -4724,7 +5898,7 @@ define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM60-NEXT: $L__BB123_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB165_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM60-NEXT: min.u16 %rs5, %rs4, %rs2;
@@ -4733,7 +5907,7 @@ define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB123_1;
+; SM60-NEXT: @%p1 bra $L__BB165_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -4759,7 +5933,7 @@ define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: $L__BB124_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB166_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -4781,7 +5955,7 @@ define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB124_1;
+; SM60-NEXT: @%p1 bra $L__BB166_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -4807,7 +5981,7 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: $L__BB125_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB167_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -4829,14 +6003,14 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB125_1;
+; SM60-NEXT: @%p1 bra $L__BB167_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM60-NEXT: $L__BB125_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB167_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -4858,7 +6032,7 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB125_3;
+; SM60-NEXT: @%p2 bra $L__BB167_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -4867,6 +6041,24 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
ret <8 x i16> %retval
}
+define <1 x i32> @umin_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: umin_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [umin_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @umin_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: umin_acq_rel_v2i32_global_cta(
; SM60: {
@@ -4934,7 +6126,24 @@ define <8 x i32> @umin_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
-define <2 x i64> @umin_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+define <1 x i64> @umin_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: umin_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @umin_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: umin_acq_rel_v2i64_global_cta(
; SM60: {
; SM60-NEXT: .reg .b64 %rd<6>;
@@ -5004,6 +6213,52 @@ define <8 x i64> @umin_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
ret <8 x i64> %retval
}
+define <1 x i8> @uinc_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<6>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [uinc_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB176_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: and.b16 %rs3, %rs2, 255;
+; SM60-NEXT: add.s16 %rs4, %rs2, 1;
+; SM60-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs5, 0, %rs4, %p1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM60-NEXT: and.b32 %r10, %r9, 255;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p2 bra $L__BB176_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: uinc_wrap_acq_rel_v2i8_global_cta(
; SM60: {
@@ -5026,7 +6281,7 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
; SM60-NEXT: mov.b32 {%rs8, %rs9}, %r4;
-; SM60-NEXT: $L__BB132_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB177_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r15, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -5048,7 +6303,7 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM60-NEXT: setp.ne.b32 %p3, %r3, %r15;
; SM60-NEXT: mov.b32 %r15, %r3;
-; SM60-NEXT: @%p3 bra $L__BB132_1;
+; SM60-NEXT: @%p3 bra $L__BB177_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r14, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -5075,7 +6330,7 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
-; SM60-NEXT: $L__BB133_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB178_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r3, %r22, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
@@ -5107,7 +6362,7 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
; SM60-NEXT: setp.ne.b32 %p5, %r1, %r22;
; SM60-NEXT: mov.b32 %r22, %r1;
-; SM60-NEXT: @%p5 bra $L__BB133_1;
+; SM60-NEXT: @%p5 bra $L__BB178_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -5133,7 +6388,7 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
-; SM60-NEXT: $L__BB134_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB179_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -5207,7 +6462,7 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB134_1;
+; SM60-NEXT: @%p9 bra $L__BB179_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -5216,6 +6471,50 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i8> %retval
}
+define <1 x i16> @uinc_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB180_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: add.s16 %rs3, %rs2, 1;
+; SM60-NEXT: setp.ge.u16 %p1, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs4, 0, %rs3, %p1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p2 bra $L__BB180_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: uinc_wrap_acq_rel_v2i16_global_cta(
; SM60: {
@@ -5230,7 +6529,7 @@ define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NEXT: $L__BB135_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB181_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; SM60-NEXT: add.s16 %rs3, %rs1, 1;
@@ -5243,7 +6542,7 @@ define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p3, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p3 bra $L__BB135_1;
+; SM60-NEXT: @%p3 bra $L__BB181_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -5269,7 +6568,7 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r1;
-; SM60-NEXT: $L__BB136_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB182_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -5299,7 +6598,7 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB136_1;
+; SM60-NEXT: @%p5 bra $L__BB182_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -5325,7 +6624,7 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r1;
-; SM60-NEXT: $L__BB137_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB183_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -5355,12 +6654,12 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB137_1;
+; SM60-NEXT: @%p5 bra $L__BB183_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: $L__BB137_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB183_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -5392,7 +6691,7 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd16;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p10 bra $L__BB137_3;
+; SM60-NEXT: @%p10 bra $L__BB183_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -5401,6 +6700,24 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i16> %retval
}
+define <1 x i32> @uinc_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [uinc_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.inc.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @uinc_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: uinc_wrap_acq_rel_v2i32_global_cta(
; SM60: {
@@ -5468,6 +6785,34 @@ define <8 x i32> @uinc_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i32> %retval
}
+define <1 x i64> @uinc_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<7>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd3];
+; SM60-NEXT: $L__BB188_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd4, %rd6, 1;
+; SM60-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM60-NEXT: selp.b64 %rd5, 0, %rd4, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM60-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
+; SM60-NEXT: mov.b64 %rd6, %rd2;
+; SM60-NEXT: @%p2 bra $L__BB188_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: uinc_wrap_acq_rel_v2i64_global_cta(
; SM60: {
@@ -5479,7 +6824,7 @@ define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
-; SM60-NEXT: $L__BB141_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB189_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd6, %rd10, 1;
; SM60-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
@@ -5487,10 +6832,10 @@ define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
; SM60-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
; SM60-NEXT: mov.b64 %rd10, %rd3;
-; SM60-NEXT: @%p2 bra $L__BB141_1;
+; SM60-NEXT: @%p2 bra $L__BB189_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
-; SM60-NEXT: $L__BB141_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB189_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd8, %rd11, 1;
; SM60-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
@@ -5498,7 +6843,7 @@ define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM60-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
; SM60-NEXT: mov.b64 %rd11, %rd4;
-; SM60-NEXT: @%p4 bra $L__BB141_3;
+; SM60-NEXT: @%p4 bra $L__BB189_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -5519,7 +6864,7 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
-; SM60-NEXT: $L__BB142_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB190_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd5, %rd18;
; SM60-NEXT: add.s64 %rd10, %rd5, 1;
@@ -5527,10 +6872,10 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: selp.b64 %rd11, 0, %rd10, %p1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
; SM60-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
-; SM60-NEXT: @%p2 bra $L__BB142_1;
+; SM60-NEXT: @%p2 bra $L__BB190_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
-; SM60-NEXT: $L__BB142_3: // %atomicrmw.start5
+; SM60-NEXT: $L__BB190_3: // %atomicrmw.start5
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd6, %rd19;
; SM60-NEXT: add.s64 %rd12, %rd6, 1;
@@ -5538,10 +6883,10 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: selp.b64 %rd13, 0, %rd12, %p3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
; SM60-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
-; SM60-NEXT: @%p4 bra $L__BB142_3;
+; SM60-NEXT: @%p4 bra $L__BB190_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end4
; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
-; SM60-NEXT: $L__BB142_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB190_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd14, %rd20, 1;
; SM60-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
@@ -5549,10 +6894,10 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
; SM60-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
; SM60-NEXT: mov.b64 %rd20, %rd7;
-; SM60-NEXT: @%p6 bra $L__BB142_5;
+; SM60-NEXT: @%p6 bra $L__BB190_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
-; SM60-NEXT: $L__BB142_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB190_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd16, %rd21, 1;
; SM60-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
@@ -5560,7 +6905,7 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM60-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
; SM60-NEXT: mov.b64 %rd21, %rd8;
-; SM60-NEXT: @%p8 bra $L__BB142_7;
+; SM60-NEXT: @%p8 bra $L__BB190_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -5584,7 +6929,7 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
-; SM60-NEXT: $L__BB143_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB191_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd7, %rd34;
; SM60-NEXT: add.s64 %rd18, %rd7, 1;
@@ -5592,10 +6937,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd19, 0, %rd18, %p1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
; SM60-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
-; SM60-NEXT: @%p2 bra $L__BB143_1;
+; SM60-NEXT: @%p2 bra $L__BB191_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
-; SM60-NEXT: $L__BB143_3: // %atomicrmw.start9
+; SM60-NEXT: $L__BB191_3: // %atomicrmw.start9
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd8, %rd35;
; SM60-NEXT: add.s64 %rd20, %rd8, 1;
@@ -5603,10 +6948,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd21, 0, %rd20, %p3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
; SM60-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
-; SM60-NEXT: @%p4 bra $L__BB143_3;
+; SM60-NEXT: @%p4 bra $L__BB191_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end8
; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
-; SM60-NEXT: $L__BB143_5: // %atomicrmw.start20
+; SM60-NEXT: $L__BB191_5: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd9, %rd36;
; SM60-NEXT: add.s64 %rd22, %rd9, 1;
@@ -5614,10 +6959,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd23, 0, %rd22, %p5;
; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
; SM60-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
-; SM60-NEXT: @%p6 bra $L__BB143_5;
+; SM60-NEXT: @%p6 bra $L__BB191_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end19
; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
-; SM60-NEXT: $L__BB143_7: // %atomicrmw.start26
+; SM60-NEXT: $L__BB191_7: // %atomicrmw.start26
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd10, %rd37;
; SM60-NEXT: add.s64 %rd24, %rd10, 1;
@@ -5625,10 +6970,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd25, 0, %rd24, %p7;
; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
; SM60-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
-; SM60-NEXT: @%p8 bra $L__BB143_7;
+; SM60-NEXT: @%p8 bra $L__BB191_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end25
; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
-; SM60-NEXT: $L__BB143_9: // %atomicrmw.start41
+; SM60-NEXT: $L__BB191_9: // %atomicrmw.start41
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd13, %rd38;
; SM60-NEXT: add.s64 %rd26, %rd13, 1;
@@ -5636,10 +6981,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd27, 0, %rd26, %p9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
; SM60-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
-; SM60-NEXT: @%p10 bra $L__BB143_9;
+; SM60-NEXT: @%p10 bra $L__BB191_9;
; SM60-NEXT: // %bb.10: // %atomicrmw.end40
; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
-; SM60-NEXT: $L__BB143_11: // %atomicrmw.start47
+; SM60-NEXT: $L__BB191_11: // %atomicrmw.start47
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd14, %rd39;
; SM60-NEXT: add.s64 %rd28, %rd14, 1;
@@ -5647,10 +6992,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd29, 0, %rd28, %p11;
; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
; SM60-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
-; SM60-NEXT: @%p12 bra $L__BB143_11;
+; SM60-NEXT: @%p12 bra $L__BB191_11;
; SM60-NEXT: // %bb.12: // %atomicrmw.end46
; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
-; SM60-NEXT: $L__BB143_13: // %atomicrmw.start58
+; SM60-NEXT: $L__BB191_13: // %atomicrmw.start58
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd30, %rd40, 1;
; SM60-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
@@ -5658,10 +7003,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
; SM60-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
; SM60-NEXT: mov.b64 %rd40, %rd15;
-; SM60-NEXT: @%p14 bra $L__BB143_13;
+; SM60-NEXT: @%p14 bra $L__BB191_13;
; SM60-NEXT: // %bb.14: // %atomicrmw.end57
; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
-; SM60-NEXT: $L__BB143_15: // %atomicrmw.start64
+; SM60-NEXT: $L__BB191_15: // %atomicrmw.start64
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd32, %rd41, 1;
; SM60-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
@@ -5669,7 +7014,7 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM60-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
; SM60-NEXT: mov.b64 %rd41, %rd16;
-; SM60-NEXT: @%p16 bra $L__BB143_15;
+; SM60-NEXT: @%p16 bra $L__BB191_15;
; SM60-NEXT: // %bb.16: // %atomicrmw.end63
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -5681,6 +7026,54 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i64> %retval
}
+define <1 x i8> @udec_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [udec_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB192_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: and.b16 %rs3, %rs2, 255;
+; SM60-NEXT: add.s16 %rs4, %rs2, -1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs3, 0;
+; SM60-NEXT: setp.gt.u16 %p2, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs5, %rs1, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-NEXT: and.b32 %r10, %r9, 255;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p3 bra $L__BB192_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: udec_wrap_acq_rel_v2i8_global_cta(
; SM60: {
@@ -5703,7 +7096,7 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
; SM60-NEXT: mov.b32 {%rs8, %rs9}, %r4;
-; SM60-NEXT: $L__BB144_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB193_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r15, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -5729,7 +7122,7 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM60-NEXT: setp.ne.b32 %p5, %r3, %r15;
; SM60-NEXT: mov.b32 %r15, %r3;
-; SM60-NEXT: @%p5 bra $L__BB144_1;
+; SM60-NEXT: @%p5 bra $L__BB193_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r14, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -5756,7 +7149,7 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
-; SM60-NEXT: $L__BB145_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB194_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r3, %r26, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
@@ -5796,7 +7189,7 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM60-NEXT: setp.ne.b32 %p9, %r1, %r26;
; SM60-NEXT: mov.b32 %r26, %r1;
-; SM60-NEXT: @%p9 bra $L__BB145_1;
+; SM60-NEXT: @%p9 bra $L__BB194_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -5824,7 +7217,7 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs17, %r11;
; SM60-NEXT: cvt.u16.u32 %rs18, %r12;
-; SM60-NEXT: $L__BB146_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB195_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -5920,7 +7313,7 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p17, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p17 bra $L__BB146_1;
+; SM60-NEXT: @%p17 bra $L__BB195_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -5929,6 +7322,52 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i8> %retval
}
+define <1 x i16> @udec_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<6>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB196_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: add.s16 %rs3, %rs2, -1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM60-NEXT: setp.gt.u16 %p2, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs4, %rs1, %rs3, %p2;
+; SM60-NEXT: selp.b16 %rs5, %rs1, %rs4, %p1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p3, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p3 bra $L__BB196_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: udec_wrap_acq_rel_v2i16_global_cta(
; SM60: {
@@ -5943,7 +7382,7 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NEXT: $L__BB147_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB197_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; SM60-NEXT: add.s16 %rs3, %rs1, -1;
@@ -5960,7 +7399,7 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p5, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p5 bra $L__BB147_1;
+; SM60-NEXT: @%p5 bra $L__BB197_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -5986,7 +7425,7 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r1;
-; SM60-NEXT: $L__BB148_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB198_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -6024,7 +7463,7 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB148_1;
+; SM60-NEXT: @%p9 bra $L__BB198_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -6050,7 +7489,7 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r1;
-; SM60-NEXT: $L__BB149_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB199_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -6088,12 +7527,12 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB149_1;
+; SM60-NEXT: @%p9 bra $L__BB199_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: $L__BB149_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB199_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -6133,7 +7572,7 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd16;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p18 bra $L__BB149_3;
+; SM60-NEXT: @%p18 bra $L__BB199_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -6142,6 +7581,24 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i16> %retval
}
+define <1 x i32> @udec_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [udec_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.dec.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @udec_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: udec_wrap_acq_rel_v2i32_global_cta(
; SM60: {
@@ -6209,6 +7666,36 @@ define <8 x i32> @udec_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i32> %retval
}
+define <1 x i64> @udec_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b64 %rd<8>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd7, [%rd3];
+; SM60-NEXT: $L__BB204_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd4, %rd7, -1;
+; SM60-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM60-NEXT: setp.gt.u64 %p2, %rd7, %rd1;
+; SM60-NEXT: selp.b64 %rd5, %rd1, %rd4, %p2;
+; SM60-NEXT: selp.b64 %rd6, %rd1, %rd5, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd7, %rd6;
+; SM60-NEXT: setp.ne.b64 %p3, %rd2, %rd7;
+; SM60-NEXT: mov.b64 %rd7, %rd2;
+; SM60-NEXT: @%p3 bra $L__BB204_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: udec_wrap_acq_rel_v2i64_global_cta(
; SM60: {
@@ -6220,7 +7707,7 @@ define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd5];
-; SM60-NEXT: $L__BB153_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB205_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd6, %rd12, -1;
; SM60-NEXT: setp.eq.b64 %p1, %rd12, 0;
@@ -6230,10 +7717,10 @@ define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd12, %rd8;
; SM60-NEXT: setp.ne.b64 %p3, %rd3, %rd12;
; SM60-NEXT: mov.b64 %rd12, %rd3;
-; SM60-NEXT: @%p3 bra $L__BB153_1;
+; SM60-NEXT: @%p3 bra $L__BB205_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd5+8];
-; SM60-NEXT: $L__BB153_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB205_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd9, %rd13, -1;
; SM60-NEXT: setp.eq.b64 %p4, %rd13, 0;
@@ -6243,7 +7730,7 @@ define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd13, %rd11;
; SM60-NEXT: setp.ne.b64 %p6, %rd4, %rd13;
; SM60-NEXT: mov.b64 %rd13, %rd4;
-; SM60-NEXT: @%p6 bra $L__BB153_3;
+; SM60-NEXT: @%p6 bra $L__BB205_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -6264,7 +7751,7 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd9];
-; SM60-NEXT: $L__BB154_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB206_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd5, %rd22;
; SM60-NEXT: add.s64 %rd10, %rd5, -1;
@@ -6274,10 +7761,10 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: selp.b64 %rd12, %rd3, %rd11, %p1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd22, [%rd9], %rd5, %rd12;
; SM60-NEXT: setp.ne.b64 %p3, %rd22, %rd5;
-; SM60-NEXT: @%p3 bra $L__BB154_1;
+; SM60-NEXT: @%p3 bra $L__BB206_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd23, [%rd9+8];
-; SM60-NEXT: $L__BB154_3: // %atomicrmw.start5
+; SM60-NEXT: $L__BB206_3: // %atomicrmw.start5
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd6, %rd23;
; SM60-NEXT: add.s64 %rd13, %rd6, -1;
@@ -6287,10 +7774,10 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: selp.b64 %rd15, %rd4, %rd14, %p4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd9+8], %rd6, %rd15;
; SM60-NEXT: setp.ne.b64 %p6, %rd23, %rd6;
-; SM60-NEXT: @%p6 bra $L__BB154_3;
+; SM60-NEXT: @%p6 bra $L__BB206_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end4
; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd9+16];
-; SM60-NEXT: $L__BB154_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB206_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd16, %rd24, -1;
; SM60-NEXT: setp.eq.b64 %p7, %rd24, 0;
@@ -6300,10 +7787,10 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd24, %rd18;
; SM60-NEXT: setp.ne.b64 %p9, %rd7, %rd24;
; SM60-NEXT: mov.b64 %rd24, %rd7;
-; SM60-NEXT: @%p9 bra $L__BB154_5;
+; SM60-NEXT: @%p9 bra $L__BB206_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd25, [%rd9+24];
-; SM60-NEXT: $L__BB154_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB206_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd19, %rd25, -1;
; SM60-NEXT: setp.eq.b64 %p10, %rd25, 0;
@@ -6313,7 +7800,7 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd25, %rd21;
; SM60-NEXT: setp.ne.b64 %p12, %rd8, %rd25;
; SM60-NEXT: mov.b64 %rd25, %rd8;
-; SM60-NEXT: @%p12 bra $L__BB154_7;
+; SM60-NEXT: @%p12 bra $L__BB206_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -6337,7 +7824,7 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd42, [%rd17];
-; SM60-NEXT: $L__BB155_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB207_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd7, %rd42;
; SM60-NEXT: add.s64 %rd18, %rd7, -1;
@@ -6347,10 +7834,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd20, %rd5, %rd19, %p1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd42, [%rd17], %rd7, %rd20;
; SM60-NEXT: setp.ne.b64 %p3, %rd42, %rd7;
-; SM60-NEXT: @%p3 bra $L__BB155_1;
+; SM60-NEXT: @%p3 bra $L__BB207_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd43, [%rd17+8];
-; SM60-NEXT: $L__BB155_3: // %atomicrmw.start9
+; SM60-NEXT: $L__BB207_3: // %atomicrmw.start9
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd8, %rd43;
; SM60-NEXT: add.s64 %rd21, %rd8, -1;
@@ -6360,10 +7847,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd23, %rd6, %rd22, %p4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd43, [%rd17+8], %rd8, %rd23;
; SM60-NEXT: setp.ne.b64 %p6, %rd43, %rd8;
-; SM60-NEXT: @%p6 bra $L__BB155_3;
+; SM60-NEXT: @%p6 bra $L__BB207_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end8
; SM60-NEXT: ld.volatile.global.b64 %rd44, [%rd17+16];
-; SM60-NEXT: $L__BB155_5: // %atomicrmw.start20
+; SM60-NEXT: $L__BB207_5: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd9, %rd44;
; SM60-NEXT: add.s64 %rd24, %rd9, -1;
@@ -6373,10 +7860,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd26, %rd3, %rd25, %p7;
; SM60-NEXT: atom.cta.global.cas.b64 %rd44, [%rd17+16], %rd9, %rd26;
; SM60-NEXT: setp.ne.b64 %p9, %rd44, %rd9;
-; SM60-NEXT: @%p9 bra $L__BB155_5;
+; SM60-NEXT: @%p9 bra $L__BB207_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end19
; SM60-NEXT: ld.volatile.global.b64 %rd45, [%rd17+24];
-; SM60-NEXT: $L__BB155_7: // %atomicrmw.start26
+; SM60-NEXT: $L__BB207_7: // %atomicrmw.start26
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd10, %rd45;
; SM60-NEXT: add.s64 %rd27, %rd10, -1;
@@ -6386,10 +7873,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd29, %rd4, %rd28, %p10;
; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd17+24], %rd10, %rd29;
; SM60-NEXT: setp.ne.b64 %p12, %rd45, %rd10;
-; SM60-NEXT: @%p12 bra $L__BB155_7;
+; SM60-NEXT: @%p12 bra $L__BB207_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end25
; SM60-NEXT: ld.volatile.global.b64 %rd46, [%rd17+32];
-; SM60-NEXT: $L__BB155_9: // %atomicrmw.start41
+; SM60-NEXT: $L__BB207_9: // %atomicrmw.start41
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd13, %rd46;
; SM60-NEXT: add.s64 %rd30, %rd13, -1;
@@ -6399,10 +7886,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd32, %rd1, %rd31, %p13;
; SM60-NEXT: atom.cta.global.cas.b64 %rd46, [%rd17+32], %rd13, %rd32;
; SM60-NEXT: setp.ne.b64 %p15, %rd46, %rd13;
-; SM60-NEXT: @%p15 bra $L__BB155_9;
+; SM60-NEXT: @%p15 bra $L__BB207_9;
; SM60-NEXT: // %bb.10: // %atomicrmw.end40
; SM60-NEXT: ld.volatile.global.b64 %rd47, [%rd17+40];
-; SM60-NEXT: $L__BB155_11: // %atomicrmw.start47
+; SM60-NEXT: $L__BB207_11: // %atomicrmw.start47
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd14, %rd47;
; SM60-NEXT: add.s64 %rd33, %rd14, -1;
@@ -6412,10 +7899,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd35, %rd2, %rd34, %p16;
; SM60-NEXT: atom.cta.global.cas.b64 %rd47, [%rd17+40], %rd14, %rd35;
; SM60-NEXT: setp.ne.b64 %p18, %rd47, %rd14;
-; SM60-NEXT: @%p18 bra $L__BB155_11;
+; SM60-NEXT: @%p18 bra $L__BB207_11;
; SM60-NEXT: // %bb.12: // %atomicrmw.end46
; SM60-NEXT: ld.volatile.global.b64 %rd48, [%rd17+48];
-; SM60-NEXT: $L__BB155_13: // %atomicrmw.start58
+; SM60-NEXT: $L__BB207_13: // %atomicrmw.start58
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd36, %rd48, -1;
; SM60-NEXT: setp.eq.b64 %p19, %rd48, 0;
@@ -6425,10 +7912,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd48, %rd38;
; SM60-NEXT: setp.ne.b64 %p21, %rd15, %rd48;
; SM60-NEXT: mov.b64 %rd48, %rd15;
-; SM60-NEXT: @%p21 bra $L__BB155_13;
+; SM60-NEXT: @%p21 bra $L__BB207_13;
; SM60-NEXT: // %bb.14: // %atomicrmw.end57
; SM60-NEXT: ld.volatile.global.b64 %rd49, [%rd17+56];
-; SM60-NEXT: $L__BB155_15: // %atomicrmw.start64
+; SM60-NEXT: $L__BB207_15: // %atomicrmw.start64
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.s64 %rd39, %rd49, -1;
; SM60-NEXT: setp.eq.b64 %p22, %rd49, 0;
@@ -6438,7 +7925,7 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd49, %rd41;
; SM60-NEXT: setp.ne.b64 %p24, %rd16, %rd49;
; SM60-NEXT: mov.b64 %rd49, %rd16;
-; SM60-NEXT: @%p24 bra $L__BB155_15;
+; SM60-NEXT: @%p24 bra $L__BB207_15;
; SM60-NEXT: // %bb.16: // %atomicrmw.end63
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -6450,6 +7937,52 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i64> %retval
}
+define <1 x i8> @usub_cond_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<6>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [usub_cond_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB208_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: and.b16 %rs3, %rs2, 255;
+; SM60-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM60-NEXT: sub.s16 %rs4, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM60-NEXT: and.b32 %r10, %r9, 255;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p2 bra $L__BB208_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: usub_cond_acq_rel_v2i8_global_cta(
; SM60: {
@@ -6472,7 +8005,7 @@ define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
; SM60-NEXT: mov.b32 {%rs6, %rs7}, %r4;
-; SM60-NEXT: $L__BB156_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB209_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r15, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -6494,7 +8027,7 @@ define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM60-NEXT: setp.ne.b32 %p3, %r3, %r15;
; SM60-NEXT: mov.b32 %r15, %r3;
-; SM60-NEXT: @%p3 bra $L__BB156_1;
+; SM60-NEXT: @%p3 bra $L__BB209_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r14, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -6525,7 +8058,7 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: cvt.u16.u32 %rs2, %r7;
; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
; SM60-NEXT: cvt.u16.u32 %rs4, %r3;
-; SM60-NEXT: $L__BB157_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB210_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r22, 0, 0x7770U;
; SM60-NEXT: setp.ge.u32 %p1, %r4, %r3;
@@ -6557,7 +8090,7 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
; SM60-NEXT: setp.ne.b32 %p5, %r1, %r22;
; SM60-NEXT: mov.b32 %r22, %r1;
-; SM60-NEXT: @%p5 bra $L__BB157_1;
+; SM60-NEXT: @%p5 bra $L__BB210_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -6585,7 +8118,7 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs13, %r5;
; SM60-NEXT: cvt.u16.u32 %rs15, %r3;
-; SM60-NEXT: $L__BB158_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB211_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -6665,7 +8198,7 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB158_1;
+; SM60-NEXT: @%p9 bra $L__BB211_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -6674,6 +8207,50 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i8> %retval
}
+define <1 x i16> @usub_cond_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB212_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: setp.ge.u16 %p1, %rs2, %rs1;
+; SM60-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs4, %rs3, %rs2, %p1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p2 bra $L__BB212_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: usub_cond_acq_rel_v2i16_global_cta(
; SM60: {
@@ -6688,7 +8265,7 @@ define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM60-NEXT: $L__BB159_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB213_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM60-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
@@ -6701,7 +8278,7 @@ define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p3, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p3 bra $L__BB159_1;
+; SM60-NEXT: @%p3 bra $L__BB213_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -6727,7 +8304,7 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; SM60-NEXT: $L__BB160_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB214_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -6757,7 +8334,7 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB160_1;
+; SM60-NEXT: @%p5 bra $L__BB214_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -6783,7 +8360,7 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; SM60-NEXT: $L__BB161_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB215_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -6813,12 +8390,12 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB161_1;
+; SM60-NEXT: @%p5 bra $L__BB215_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: $L__BB161_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB215_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -6850,7 +8427,7 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd16;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p10 bra $L__BB161_3;
+; SM60-NEXT: @%p10 bra $L__BB215_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -6859,6 +8436,35 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i16> %retval
}
+define <1 x i32> @usub_cond_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [usub_cond_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB216_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u32 %p1, %r5, %r1;
+; SM60-NEXT: sub.s32 %r3, %r5, %r1;
+; SM60-NEXT: selp.b32 %r4, %r3, %r5, %p1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p2, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p2 bra $L__BB216_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: usub_cond_acq_rel_v2i32_global_cta(
; SM60: {
@@ -6873,7 +8479,7 @@ define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: $L__BB162_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB217_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -6893,7 +8499,7 @@ define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM60-NEXT: @%p3 bra $L__BB162_1;
+; SM60-NEXT: @%p3 bra $L__BB217_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -6916,7 +8522,7 @@ define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
-; SM60-NEXT: $L__BB163_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB218_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
; SM60-NEXT: cvt.u64.u32 %rd4, %r14;
@@ -6936,12 +8542,12 @@ define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM60-NEXT: @%p3 bra $L__BB163_1;
+; SM60-NEXT: @%p3 bra $L__BB218_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r15, %rd12;
-; SM60-NEXT: $L__BB163_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB218_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd13, %r15;
; SM60-NEXT: cvt.u64.u32 %rd14, %r16;
@@ -6961,7 +8567,7 @@ define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: setp.ne.b64 %p6, %rd21, %rd16;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r15, %rd21;
-; SM60-NEXT: @%p6 bra $L__BB163_3;
+; SM60-NEXT: @%p6 bra $L__BB218_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
@@ -6985,7 +8591,7 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
-; SM60-NEXT: $L__BB164_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB219_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
; SM60-NEXT: cvt.u64.u32 %rd4, %r26;
@@ -7005,12 +8611,12 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
-; SM60-NEXT: @%p3 bra $L__BB164_1;
+; SM60-NEXT: @%p3 bra $L__BB219_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
-; SM60-NEXT: $L__BB164_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB219_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd13, %r27;
; SM60-NEXT: cvt.u64.u32 %rd14, %r28;
@@ -7030,12 +8636,12 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p6, %rd21, %rd16;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
-; SM60-NEXT: @%p6 bra $L__BB164_3;
+; SM60-NEXT: @%p6 bra $L__BB219_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
; SM60-NEXT: cvt.u32.u64 %r29, %rd22;
-; SM60-NEXT: $L__BB164_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB219_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd23, %r29;
; SM60-NEXT: cvt.u64.u32 %rd24, %r30;
@@ -7055,12 +8661,12 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p9, %rd31, %rd26;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
; SM60-NEXT: cvt.u32.u64 %r29, %rd31;
-; SM60-NEXT: @%p9 bra $L__BB164_5;
+; SM60-NEXT: @%p9 bra $L__BB219_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
; SM60-NEXT: cvt.u32.u64 %r31, %rd32;
-; SM60-NEXT: $L__BB164_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB219_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd33, %r31;
; SM60-NEXT: cvt.u64.u32 %rd34, %r32;
@@ -7080,7 +8686,7 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p12, %rd41, %rd36;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
; SM60-NEXT: cvt.u32.u64 %r31, %rd41;
-; SM60-NEXT: @%p12 bra $L__BB164_7;
+; SM60-NEXT: @%p12 bra $L__BB219_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
@@ -7090,6 +8696,34 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i32> %retval
}
+define <1 x i64> @usub_cond_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<7>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd3];
+; SM60-NEXT: $L__BB220_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM60-NEXT: sub.s64 %rd4, %rd6, %rd1;
+; SM60-NEXT: selp.b64 %rd5, %rd4, %rd6, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM60-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
+; SM60-NEXT: mov.b64 %rd6, %rd2;
+; SM60-NEXT: @%p2 bra $L__BB220_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: usub_cond_acq_rel_v2i64_global_cta(
; SM60: {
@@ -7101,7 +8735,7 @@ define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
-; SM60-NEXT: $L__BB165_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB221_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
; SM60-NEXT: sub.s64 %rd6, %rd10, %rd1;
@@ -7109,10 +8743,10 @@ define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
; SM60-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
; SM60-NEXT: mov.b64 %rd10, %rd3;
-; SM60-NEXT: @%p2 bra $L__BB165_1;
+; SM60-NEXT: @%p2 bra $L__BB221_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
-; SM60-NEXT: $L__BB165_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB221_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
; SM60-NEXT: sub.s64 %rd8, %rd11, %rd2;
@@ -7120,7 +8754,7 @@ define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM60-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
; SM60-NEXT: mov.b64 %rd11, %rd4;
-; SM60-NEXT: @%p4 bra $L__BB165_3;
+; SM60-NEXT: @%p4 bra $L__BB221_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -7141,7 +8775,7 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
-; SM60-NEXT: $L__BB166_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB222_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd5, %rd18;
; SM60-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
@@ -7149,10 +8783,10 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: selp.b64 %rd11, %rd10, %rd5, %p1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
; SM60-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
-; SM60-NEXT: @%p2 bra $L__BB166_1;
+; SM60-NEXT: @%p2 bra $L__BB222_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
-; SM60-NEXT: $L__BB166_3: // %atomicrmw.start5
+; SM60-NEXT: $L__BB222_3: // %atomicrmw.start5
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd6, %rd19;
; SM60-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
@@ -7160,10 +8794,10 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: selp.b64 %rd13, %rd12, %rd6, %p3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
; SM60-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
-; SM60-NEXT: @%p4 bra $L__BB166_3;
+; SM60-NEXT: @%p4 bra $L__BB222_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end4
; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
-; SM60-NEXT: $L__BB166_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB222_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
; SM60-NEXT: sub.s64 %rd14, %rd20, %rd1;
@@ -7171,10 +8805,10 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
; SM60-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
; SM60-NEXT: mov.b64 %rd20, %rd7;
-; SM60-NEXT: @%p6 bra $L__BB166_5;
+; SM60-NEXT: @%p6 bra $L__BB222_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
-; SM60-NEXT: $L__BB166_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB222_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
; SM60-NEXT: sub.s64 %rd16, %rd21, %rd2;
@@ -7182,7 +8816,7 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM60-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
; SM60-NEXT: mov.b64 %rd21, %rd8;
-; SM60-NEXT: @%p8 bra $L__BB166_7;
+; SM60-NEXT: @%p8 bra $L__BB222_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -7206,7 +8840,7 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
-; SM60-NEXT: $L__BB167_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB223_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd7, %rd34;
; SM60-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
@@ -7214,10 +8848,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd19, %rd18, %rd7, %p1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
; SM60-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
-; SM60-NEXT: @%p2 bra $L__BB167_1;
+; SM60-NEXT: @%p2 bra $L__BB223_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
-; SM60-NEXT: $L__BB167_3: // %atomicrmw.start9
+; SM60-NEXT: $L__BB223_3: // %atomicrmw.start9
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd8, %rd35;
; SM60-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
@@ -7225,10 +8859,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd21, %rd20, %rd8, %p3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
; SM60-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
-; SM60-NEXT: @%p4 bra $L__BB167_3;
+; SM60-NEXT: @%p4 bra $L__BB223_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end8
; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
-; SM60-NEXT: $L__BB167_5: // %atomicrmw.start20
+; SM60-NEXT: $L__BB223_5: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd9, %rd36;
; SM60-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
@@ -7236,10 +8870,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd23, %rd22, %rd9, %p5;
; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
; SM60-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
-; SM60-NEXT: @%p6 bra $L__BB167_5;
+; SM60-NEXT: @%p6 bra $L__BB223_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end19
; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
-; SM60-NEXT: $L__BB167_7: // %atomicrmw.start26
+; SM60-NEXT: $L__BB223_7: // %atomicrmw.start26
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd10, %rd37;
; SM60-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
@@ -7247,10 +8881,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd25, %rd24, %rd10, %p7;
; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
; SM60-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
-; SM60-NEXT: @%p8 bra $L__BB167_7;
+; SM60-NEXT: @%p8 bra $L__BB223_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end25
; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
-; SM60-NEXT: $L__BB167_9: // %atomicrmw.start41
+; SM60-NEXT: $L__BB223_9: // %atomicrmw.start41
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd13, %rd38;
; SM60-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
@@ -7258,10 +8892,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd27, %rd26, %rd13, %p9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
; SM60-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
-; SM60-NEXT: @%p10 bra $L__BB167_9;
+; SM60-NEXT: @%p10 bra $L__BB223_9;
; SM60-NEXT: // %bb.10: // %atomicrmw.end40
; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
-; SM60-NEXT: $L__BB167_11: // %atomicrmw.start47
+; SM60-NEXT: $L__BB223_11: // %atomicrmw.start47
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd14, %rd39;
; SM60-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
@@ -7269,10 +8903,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: selp.b64 %rd29, %rd28, %rd14, %p11;
; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
; SM60-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
-; SM60-NEXT: @%p12 bra $L__BB167_11;
+; SM60-NEXT: @%p12 bra $L__BB223_11;
; SM60-NEXT: // %bb.12: // %atomicrmw.end46
; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
-; SM60-NEXT: $L__BB167_13: // %atomicrmw.start58
+; SM60-NEXT: $L__BB223_13: // %atomicrmw.start58
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
; SM60-NEXT: sub.s64 %rd30, %rd40, %rd11;
@@ -7280,10 +8914,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
; SM60-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
; SM60-NEXT: mov.b64 %rd40, %rd15;
-; SM60-NEXT: @%p14 bra $L__BB167_13;
+; SM60-NEXT: @%p14 bra $L__BB223_13;
; SM60-NEXT: // %bb.14: // %atomicrmw.end57
; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
-; SM60-NEXT: $L__BB167_15: // %atomicrmw.start64
+; SM60-NEXT: $L__BB223_15: // %atomicrmw.start64
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
; SM60-NEXT: sub.s64 %rd32, %rd41, %rd12;
@@ -7291,7 +8925,7 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM60-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
; SM60-NEXT: mov.b64 %rd41, %rd16;
-; SM60-NEXT: @%p16 bra $L__BB167_15;
+; SM60-NEXT: @%p16 bra $L__BB223_15;
; SM60-NEXT: // %bb.16: // %atomicrmw.end63
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -7303,6 +8937,50 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i64> %retval
}
+define <1 x i8> @usub_sat_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<6>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [usub_sat_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB224_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: and.b16 %rs3, %rs2, 255;
+; SM60-NEXT: max.u16 %rs4, %rs3, %rs1;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB224_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: usub_sat_acq_rel_v2i8_global_cta(
; SM60: {
@@ -7325,7 +9003,7 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r21, [%rd1];
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r4;
-; SM60-NEXT: $L__BB168_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB225_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r21, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -7353,7 +9031,7 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r21, %r19;
; SM60-NEXT: setp.ne.b32 %p3, %r3, %r21;
; SM60-NEXT: mov.b32 %r21, %r3;
-; SM60-NEXT: @%p3 bra $L__BB168_1;
+; SM60-NEXT: @%p3 bra $L__BB225_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r20, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -7384,7 +9062,7 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
; SM60-NEXT: cvt.u16.u32 %rs9, %r10;
; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs13, %r13;
-; SM60-NEXT: $L__BB169_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB226_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -7412,7 +9090,7 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM60-NEXT: mov.b32 %r18, %r1;
-; SM60-NEXT: @%p1 bra $L__BB169_1;
+; SM60-NEXT: @%p1 bra $L__BB226_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -7446,7 +9124,7 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: cvt.u16.u32 %rs13, %r13;
; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM60-NEXT: $L__BB170_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB227_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -7512,7 +9190,7 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB170_1;
+; SM60-NEXT: @%p1 bra $L__BB227_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -7521,6 +9199,49 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
ret <8 x i8> %retval
}
+define <1 x i16> @usub_sat_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB228_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM60-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB228_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM60-LABEL: usub_sat_acq_rel_v2i16_global_cta(
; SM60: {
@@ -7535,7 +9256,7 @@ define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r7, [%rd1];
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM60-NEXT: $L__BB171_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB229_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r7;
; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -7551,7 +9272,7 @@ define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
; SM60-NEXT: setp.ne.b32 %p3, %r1, %r7;
; SM60-NEXT: mov.b32 %r7, %r1;
-; SM60-NEXT: @%p3 bra $L__BB171_1;
+; SM60-NEXT: @%p3 bra $L__BB229_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -7577,7 +9298,7 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: cvt.u32.u64 %r11, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
-; SM60-NEXT: $L__BB172_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB230_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r11;
; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -7613,7 +9334,7 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB172_1;
+; SM60-NEXT: @%p5 bra $L__BB230_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r11, %r12};
@@ -7639,7 +9360,7 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: cvt.u32.u64 %r21, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
-; SM60-NEXT: $L__BB173_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB231_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r21;
; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -7675,12 +9396,12 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB173_1;
+; SM60-NEXT: @%p5 bra $L__BB231_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd12;
-; SM60-NEXT: $L__BB173_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB231_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r3;
; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r23;
@@ -7718,7 +9439,7 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd21;
-; SM60-NEXT: @%p10 bra $L__BB173_3;
+; SM60-NEXT: @%p10 bra $L__BB231_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
@@ -7727,6 +9448,34 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i16> %retval
}
+define <1 x i32> @usub_sat_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [usub_sat_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB232_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r3, %r5, %r1;
+; SM60-NEXT: sub.s32 %r4, %r3, %r1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p1 bra $L__BB232_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: usub_sat_acq_rel_v2i32_global_cta(
; SM60: {
@@ -7741,7 +9490,7 @@ define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: $L__BB174_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB233_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.u32 %r3, %r7, %r1;
; SM60-NEXT: sub.s32 %r4, %r3, %r1;
@@ -7759,7 +9508,7 @@ define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB174_1;
+; SM60-NEXT: @%p1 bra $L__BB233_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -7782,7 +9531,7 @@ define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
-; SM60-NEXT: $L__BB175_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB234_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.u32 %r5, %r13, %r1;
; SM60-NEXT: sub.s32 %r6, %r5, %r1;
@@ -7800,12 +9549,12 @@ define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB175_1;
+; SM60-NEXT: @%p1 bra $L__BB234_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r15, %rd12;
-; SM60-NEXT: $L__BB175_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB234_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.u32 %r9, %r15, %r3;
; SM60-NEXT: sub.s32 %r10, %r9, %r3;
@@ -7823,7 +9572,7 @@ define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r15, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB175_3;
+; SM60-NEXT: @%p2 bra $L__BB234_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
@@ -7847,7 +9596,7 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
-; SM60-NEXT: $L__BB176_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB235_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.u32 %r9, %r25, %r5;
; SM60-NEXT: sub.s32 %r10, %r9, %r5;
@@ -7865,12 +9614,12 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB176_1;
+; SM60-NEXT: @%p1 bra $L__BB235_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
-; SM60-NEXT: $L__BB176_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB235_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.u32 %r13, %r27, %r7;
; SM60-NEXT: sub.s32 %r14, %r13, %r7;
@@ -7888,12 +9637,12 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB176_3;
+; SM60-NEXT: @%p2 bra $L__BB235_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
; SM60-NEXT: cvt.u32.u64 %r29, %rd22;
-; SM60-NEXT: $L__BB176_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB235_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.u32 %r17, %r29, %r1;
; SM60-NEXT: sub.s32 %r18, %r17, %r1;
@@ -7911,12 +9660,12 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
; SM60-NEXT: cvt.u32.u64 %r29, %rd31;
-; SM60-NEXT: @%p3 bra $L__BB176_5;
+; SM60-NEXT: @%p3 bra $L__BB235_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
; SM60-NEXT: cvt.u32.u64 %r31, %rd32;
-; SM60-NEXT: $L__BB176_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB235_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.u32 %r21, %r31, %r3;
; SM60-NEXT: sub.s32 %r22, %r21, %r3;
@@ -7934,7 +9683,7 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
; SM60-NEXT: cvt.u32.u64 %r31, %rd41;
-; SM60-NEXT: @%p4 bra $L__BB176_7;
+; SM60-NEXT: @%p4 bra $L__BB235_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
@@ -7944,6 +9693,33 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i32> %retval
}
+define <1 x i64> @usub_sat_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b64 %rd<7>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd3];
+; SM60-NEXT: $L__BB236_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd4, %rd6, %rd1;
+; SM60-NEXT: sub.s64 %rd5, %rd4, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
+; SM60-NEXT: mov.b64 %rd6, %rd2;
+; SM60-NEXT: @%p1 bra $L__BB236_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM60-LABEL: usub_sat_acq_rel_v2i64_global_cta(
; SM60: {
@@ -7955,24 +9731,24 @@ define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
-; SM60-NEXT: $L__BB177_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB237_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.u64 %rd6, %rd10, %rd1;
; SM60-NEXT: sub.s64 %rd7, %rd6, %rd1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
; SM60-NEXT: mov.b64 %rd10, %rd3;
-; SM60-NEXT: @%p1 bra $L__BB177_1;
+; SM60-NEXT: @%p1 bra $L__BB237_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
-; SM60-NEXT: $L__BB177_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB237_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.u64 %rd8, %rd11, %rd2;
; SM60-NEXT: sub.s64 %rd9, %rd8, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
; SM60-NEXT: mov.b64 %rd11, %rd4;
-; SM60-NEXT: @%p2 bra $L__BB177_3;
+; SM60-NEXT: @%p2 bra $L__BB237_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -7993,44 +9769,44 @@ define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
-; SM60-NEXT: $L__BB178_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB238_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd5, %rd18;
; SM60-NEXT: max.u64 %rd10, %rd5, %rd3;
; SM60-NEXT: sub.s64 %rd11, %rd10, %rd3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
; SM60-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
-; SM60-NEXT: @%p1 bra $L__BB178_1;
+; SM60-NEXT: @%p1 bra $L__BB238_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
-; SM60-NEXT: $L__BB178_3: // %atomicrmw.start5
+; SM60-NEXT: $L__BB238_3: // %atomicrmw.start5
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd6, %rd19;
; SM60-NEXT: max.u64 %rd12, %rd6, %rd4;
; SM60-NEXT: sub.s64 %rd13, %rd12, %rd4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
; SM60-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
-; SM60-NEXT: @%p2 bra $L__BB178_3;
+; SM60-NEXT: @%p2 bra $L__BB238_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end4
; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
-; SM60-NEXT: $L__BB178_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB238_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.u64 %rd14, %rd20, %rd1;
; SM60-NEXT: sub.s64 %rd15, %rd14, %rd1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
; SM60-NEXT: mov.b64 %rd20, %rd7;
-; SM60-NEXT: @%p3 bra $L__BB178_5;
+; SM60-NEXT: @%p3 bra $L__BB238_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
-; SM60-NEXT: $L__BB178_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB238_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.u64 %rd16, %rd21, %rd2;
; SM60-NEXT: sub.s64 %rd17, %rd16, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
; SM60-NEXT: mov.b64 %rd21, %rd8;
-; SM60-NEXT: @%p4 bra $L__BB178_7;
+; SM60-NEXT: @%p4 bra $L__BB238_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -8054,84 +9830,84 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
-; SM60-NEXT: $L__BB179_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB239_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd7, %rd34;
; SM60-NEXT: max.u64 %rd18, %rd7, %rd5;
; SM60-NEXT: sub.s64 %rd19, %rd18, %rd5;
; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
; SM60-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
-; SM60-NEXT: @%p1 bra $L__BB179_1;
+; SM60-NEXT: @%p1 bra $L__BB239_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
-; SM60-NEXT: $L__BB179_3: // %atomicrmw.start9
+; SM60-NEXT: $L__BB239_3: // %atomicrmw.start9
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd8, %rd35;
; SM60-NEXT: max.u64 %rd20, %rd8, %rd6;
; SM60-NEXT: sub.s64 %rd21, %rd20, %rd6;
; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
; SM60-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
-; SM60-NEXT: @%p2 bra $L__BB179_3;
+; SM60-NEXT: @%p2 bra $L__BB239_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end8
; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
-; SM60-NEXT: $L__BB179_5: // %atomicrmw.start20
+; SM60-NEXT: $L__BB239_5: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd9, %rd36;
; SM60-NEXT: max.u64 %rd22, %rd9, %rd3;
; SM60-NEXT: sub.s64 %rd23, %rd22, %rd3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
; SM60-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
-; SM60-NEXT: @%p3 bra $L__BB179_5;
+; SM60-NEXT: @%p3 bra $L__BB239_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end19
; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
-; SM60-NEXT: $L__BB179_7: // %atomicrmw.start26
+; SM60-NEXT: $L__BB239_7: // %atomicrmw.start26
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd10, %rd37;
; SM60-NEXT: max.u64 %rd24, %rd10, %rd4;
; SM60-NEXT: sub.s64 %rd25, %rd24, %rd4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
; SM60-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
-; SM60-NEXT: @%p4 bra $L__BB179_7;
+; SM60-NEXT: @%p4 bra $L__BB239_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end25
; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
-; SM60-NEXT: $L__BB179_9: // %atomicrmw.start41
+; SM60-NEXT: $L__BB239_9: // %atomicrmw.start41
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd13, %rd38;
; SM60-NEXT: max.u64 %rd26, %rd13, %rd1;
; SM60-NEXT: sub.s64 %rd27, %rd26, %rd1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
; SM60-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
-; SM60-NEXT: @%p5 bra $L__BB179_9;
+; SM60-NEXT: @%p5 bra $L__BB239_9;
; SM60-NEXT: // %bb.10: // %atomicrmw.end40
; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
-; SM60-NEXT: $L__BB179_11: // %atomicrmw.start47
+; SM60-NEXT: $L__BB239_11: // %atomicrmw.start47
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd14, %rd39;
; SM60-NEXT: max.u64 %rd28, %rd14, %rd2;
; SM60-NEXT: sub.s64 %rd29, %rd28, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
; SM60-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
-; SM60-NEXT: @%p6 bra $L__BB179_11;
+; SM60-NEXT: @%p6 bra $L__BB239_11;
; SM60-NEXT: // %bb.12: // %atomicrmw.end46
; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
-; SM60-NEXT: $L__BB179_13: // %atomicrmw.start58
+; SM60-NEXT: $L__BB239_13: // %atomicrmw.start58
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.u64 %rd30, %rd40, %rd11;
; SM60-NEXT: sub.s64 %rd31, %rd30, %rd11;
; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
; SM60-NEXT: mov.b64 %rd40, %rd15;
-; SM60-NEXT: @%p7 bra $L__BB179_13;
+; SM60-NEXT: @%p7 bra $L__BB239_13;
; SM60-NEXT: // %bb.14: // %atomicrmw.end57
; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
-; SM60-NEXT: $L__BB179_15: // %atomicrmw.start64
+; SM60-NEXT: $L__BB239_15: // %atomicrmw.start64
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.u64 %rd32, %rd41, %rd12;
; SM60-NEXT: sub.s64 %rd33, %rd32, %rd12;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
; SM60-NEXT: mov.b64 %rd41, %rd16;
-; SM60-NEXT: @%p8 bra $L__BB179_15;
+; SM60-NEXT: @%p8 bra $L__BB239_15;
; SM60-NEXT: // %bb.16: // %atomicrmw.end63
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -8143,6 +9919,24 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i64> %retval
}
+define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM60-LABEL: fadd_acq_rel_v1f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM60-LABEL: fadd_acq_rel_v2f32_global_cta(
; SM60: {
@@ -8210,6 +10004,33 @@ define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
ret <8 x float> %retval
}
+define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM60-LABEL: fsub_acq_rel_v1f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB244_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r3, %r4, %r1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM60-NEXT: mov.b32 %r4, %r2;
+; SM60-NEXT: @%p1 bra $L__BB244_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM60-LABEL: fsub_acq_rel_v2f32_global_cta(
; SM60: {
@@ -8224,7 +10045,7 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB183_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB245_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f32 %r3, %r5, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -8240,7 +10061,7 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB183_1;
+; SM60-NEXT: @%p1 bra $L__BB245_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -8263,7 +10084,7 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB184_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB246_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f32 %r5, %r9, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -8279,12 +10100,12 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB184_1;
+; SM60-NEXT: @%p1 bra $L__BB246_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: $L__BB184_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB246_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f32 %r7, %r11, %r3;
; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -8300,7 +10121,7 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB184_3;
+; SM60-NEXT: @%p2 bra $L__BB246_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -8324,7 +10145,7 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM60-NEXT: $L__BB185_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB247_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f32 %r9, %r17, %r5;
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
@@ -8340,12 +10161,12 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB185_1;
+; SM60-NEXT: @%p1 bra $L__BB247_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd12;
-; SM60-NEXT: $L__BB185_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB247_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f32 %r11, %r19, %r7;
; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
@@ -8361,12 +10182,12 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB185_3;
+; SM60-NEXT: @%p2 bra $L__BB247_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd22;
-; SM60-NEXT: $L__BB185_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB247_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f32 %r13, %r21, %r1;
; SM60-NEXT: cvt.u64.u32 %rd23, %r13;
@@ -8382,12 +10203,12 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd31;
-; SM60-NEXT: @%p3 bra $L__BB185_5;
+; SM60-NEXT: @%p3 bra $L__BB247_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd32;
-; SM60-NEXT: $L__BB185_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB247_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f32 %r15, %r23, %r3;
; SM60-NEXT: cvt.u64.u32 %rd33, %r15;
@@ -8403,7 +10224,7 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd41;
-; SM60-NEXT: @%p4 bra $L__BB185_7;
+; SM60-NEXT: @%p4 bra $L__BB247_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -8413,6 +10234,33 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
ret <8 x float> %retval
}
+define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM60-LABEL: fmin_acq_rel_v1f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB248_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r3, %r4, %r1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM60-NEXT: mov.b32 %r4, %r2;
+; SM60-NEXT: @%p1 bra $L__BB248_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM60-LABEL: fmin_acq_rel_v2f32_global_cta(
; SM60: {
@@ -8427,7 +10275,7 @@ define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB186_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB249_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: min.f32 %r3, %r5, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -8443,7 +10291,7 @@ define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB186_1;
+; SM60-NEXT: @%p1 bra $L__BB249_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -8466,7 +10314,7 @@ define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB187_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB250_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: min.f32 %r5, %r9, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -8482,12 +10330,12 @@ define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB187_1;
+; SM60-NEXT: @%p1 bra $L__BB250_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: $L__BB187_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB250_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: min.f32 %r7, %r11, %r3;
; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -8503,7 +10351,7 @@ define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB187_3;
+; SM60-NEXT: @%p2 bra $L__BB250_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -8527,7 +10375,7 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM60-NEXT: $L__BB188_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB251_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: min.f32 %r9, %r17, %r5;
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
@@ -8543,12 +10391,12 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB188_1;
+; SM60-NEXT: @%p1 bra $L__BB251_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd12;
-; SM60-NEXT: $L__BB188_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB251_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: min.f32 %r11, %r19, %r7;
; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
@@ -8564,12 +10412,12 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB188_3;
+; SM60-NEXT: @%p2 bra $L__BB251_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd22;
-; SM60-NEXT: $L__BB188_5: // %atomicrmw.start15
+; SM60-NEXT: $L__BB251_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: min.f32 %r13, %r21, %r1;
; SM60-NEXT: cvt.u64.u32 %rd23, %r13;
@@ -8585,12 +10433,12 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd31;
-; SM60-NEXT: @%p3 bra $L__BB188_5;
+; SM60-NEXT: @%p3 bra $L__BB251_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd32;
-; SM60-NEXT: $L__BB188_7: // %atomicrmw.start20
+; SM60-NEXT: $L__BB251_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: min.f32 %r15, %r23, %r3;
; SM60-NEXT: cvt.u64.u32 %rd33, %r15;
@@ -8606,7 +10454,7 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd41;
-; SM60-NEXT: @%p4 bra $L__BB188_7;
+; SM60-NEXT: @%p4 bra $L__BB251_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -8616,6 +10464,33 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
ret <8 x float> %retval
}
+define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM60-LABEL: fmax_acq_rel_v1f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB252_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r3, %r4, %r1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM60-NEXT: mov.b32 %r4, %r2;
+; SM60-NEXT: @%p1 bra $L__BB252_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM60-LABEL: fmax_acq_rel_v2f32_global_cta(
; SM60: {
@@ -8630,7 +10505,7 @@ define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB189_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB253_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.f32 %r3, %r5, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -8646,7 +10521,7 @@ define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB189_1;
+; SM60-NEXT: @%p1 bra $L__BB253_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -8669,7 +10544,7 @@ define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB190_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB254_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.f32 %r5, %r9, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -8685,12 +10560,12 @@ define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB190_1;
+; SM60-NEXT: @%p1 bra $L__BB254_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: $L__BB190_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB254_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.f32 %r7, %r11, %r3;
; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -8706,7 +10581,7 @@ define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB190_3;
+; SM60-NEXT: @%p2 bra $L__BB254_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -8730,7 +10605,7 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM60-NEXT: $L__BB191_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB255_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.f32 %r9, %r17, %r5;
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
@@ -8746,12 +10621,12 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB191_1;
+; SM60-NEXT: @%p1 bra $L__BB255_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd12;
-; SM60-NEXT: $L__BB191_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB255_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.f32 %r11, %r19, %r7;
; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
@@ -8767,12 +10642,12 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB191_3;
+; SM60-NEXT: @%p2 bra $L__BB255_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd22;
-; SM60-NEXT: $L__BB191_5: // %atomicrmw.start15
+; SM60-NEXT: $L__BB255_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.f32 %r13, %r21, %r1;
; SM60-NEXT: cvt.u64.u32 %rd23, %r13;
@@ -8788,12 +10663,12 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd31;
-; SM60-NEXT: @%p3 bra $L__BB191_5;
+; SM60-NEXT: @%p3 bra $L__BB255_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd32;
-; SM60-NEXT: $L__BB191_7: // %atomicrmw.start20
+; SM60-NEXT: $L__BB255_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.f32 %r15, %r23, %r3;
; SM60-NEXT: cvt.u64.u32 %rd33, %r15;
@@ -8809,7 +10684,7 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM60-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd41;
-; SM60-NEXT: @%p4 bra $L__BB191_7;
+; SM60-NEXT: @%p4 bra $L__BB255_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -8819,6 +10694,41 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
ret <8 x float> %retval
}
+define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM60-LABEL: fminimum_acq_rel_v1f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r8, [%rd1];
+; SM60-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM60-NEXT: $L__BB256_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r8, %r1;
+; SM60-NEXT: min.f32 %r3, %r8, %r1;
+; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r8, -2147483648;
+; SM60-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM60-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r8, %r7;
+; SM60-NEXT: setp.ne.b32 %p5, %r2, %r8;
+; SM60-NEXT: mov.b32 %r8, %r2;
+; SM60-NEXT: @%p5 bra $L__BB256_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM60-LABEL: fminimum_acq_rel_v2f32_global_cta(
; SM60: {
@@ -8835,7 +10745,7 @@ define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
; SM60-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
; SM60-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
-; SM60-NEXT: $L__BB192_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB257_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p1, %r13, %r1;
; SM60-NEXT: min.f32 %r3, %r13, %r1;
@@ -8865,7 +10775,7 @@ define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB192_1;
+; SM60-NEXT: @%p9 bra $L__BB257_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
@@ -8890,7 +10800,7 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
; SM60-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
; SM60-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
-; SM60-NEXT: $L__BB193_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB258_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p1, %r25, %r1;
; SM60-NEXT: min.f32 %r5, %r25, %r1;
@@ -8920,14 +10830,14 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB193_1;
+; SM60-NEXT: @%p9 bra $L__BB258_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
; SM60-NEXT: setp.eq.b32 %p12, %r3, -2147483648;
; SM60-NEXT: setp.eq.b32 %p16, %r4, -2147483648;
-; SM60-NEXT: $L__BB193_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB258_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p10, %r27, %r3;
; SM60-NEXT: min.f32 %r15, %r27, %r3;
@@ -8957,7 +10867,7 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
-; SM60-NEXT: @%p18 bra $L__BB193_3;
+; SM60-NEXT: @%p18 bra $L__BB258_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
@@ -8983,7 +10893,7 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
; SM60-NEXT: setp.eq.b32 %p3, %r5, -2147483648;
; SM60-NEXT: setp.eq.b32 %p7, %r6, -2147483648;
-; SM60-NEXT: $L__BB194_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB259_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p1, %r49, %r5;
; SM60-NEXT: min.f32 %r9, %r49, %r5;
@@ -9013,14 +10923,14 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB194_1;
+; SM60-NEXT: @%p9 bra $L__BB259_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r51, %rd12;
; SM60-NEXT: setp.eq.b32 %p12, %r7, -2147483648;
; SM60-NEXT: setp.eq.b32 %p16, %r8, -2147483648;
-; SM60-NEXT: $L__BB194_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB259_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p10, %r51, %r7;
; SM60-NEXT: min.f32 %r19, %r51, %r7;
@@ -9050,14 +10960,14 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r51, %rd21;
-; SM60-NEXT: @%p18 bra $L__BB194_3;
+; SM60-NEXT: @%p18 bra $L__BB259_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
; SM60-NEXT: cvt.u32.u64 %r53, %rd22;
; SM60-NEXT: setp.eq.b32 %p21, %r1, -2147483648;
; SM60-NEXT: setp.eq.b32 %p25, %r2, -2147483648;
-; SM60-NEXT: $L__BB194_5: // %atomicrmw.start15
+; SM60-NEXT: $L__BB259_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p19, %r53, %r1;
; SM60-NEXT: min.f32 %r29, %r53, %r1;
@@ -9087,14 +10997,14 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
; SM60-NEXT: cvt.u32.u64 %r53, %rd31;
-; SM60-NEXT: @%p27 bra $L__BB194_5;
+; SM60-NEXT: @%p27 bra $L__BB259_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
; SM60-NEXT: cvt.u32.u64 %r55, %rd32;
; SM60-NEXT: setp.eq.b32 %p30, %r3, -2147483648;
; SM60-NEXT: setp.eq.b32 %p34, %r4, -2147483648;
-; SM60-NEXT: $L__BB194_7: // %atomicrmw.start20
+; SM60-NEXT: $L__BB259_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p28, %r55, %r3;
; SM60-NEXT: min.f32 %r39, %r55, %r3;
@@ -9124,7 +11034,7 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
; SM60-NEXT: cvt.u32.u64 %r55, %rd41;
-; SM60-NEXT: @%p36 bra $L__BB194_7;
+; SM60-NEXT: @%p36 bra $L__BB259_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
@@ -9134,6 +11044,41 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x float> %retval
}
+define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v1f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r8, [%rd1];
+; SM60-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM60-NEXT: $L__BB260_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r8, %r1;
+; SM60-NEXT: max.f32 %r3, %r8, %r1;
+; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r8, 0;
+; SM60-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM60-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r8, %r7;
+; SM60-NEXT: setp.ne.b32 %p5, %r2, %r8;
+; SM60-NEXT: mov.b32 %r8, %r2;
+; SM60-NEXT: @%p5 bra $L__BB260_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM60-LABEL: fmaximum_acq_rel_v2f32_global_cta(
; SM60: {
@@ -9150,7 +11095,7 @@ define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
; SM60-NEXT: setp.eq.b32 %p3, %r1, 0;
; SM60-NEXT: setp.eq.b32 %p7, %r2, 0;
-; SM60-NEXT: $L__BB195_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB261_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p1, %r13, %r1;
; SM60-NEXT: max.f32 %r3, %r13, %r1;
@@ -9180,7 +11125,7 @@ define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB195_1;
+; SM60-NEXT: @%p9 bra $L__BB261_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
@@ -9205,7 +11150,7 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
; SM60-NEXT: setp.eq.b32 %p3, %r1, 0;
; SM60-NEXT: setp.eq.b32 %p7, %r2, 0;
-; SM60-NEXT: $L__BB196_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB262_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p1, %r25, %r1;
; SM60-NEXT: max.f32 %r5, %r25, %r1;
@@ -9235,14 +11180,14 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB196_1;
+; SM60-NEXT: @%p9 bra $L__BB262_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
; SM60-NEXT: setp.eq.b32 %p12, %r3, 0;
; SM60-NEXT: setp.eq.b32 %p16, %r4, 0;
-; SM60-NEXT: $L__BB196_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB262_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p10, %r27, %r3;
; SM60-NEXT: max.f32 %r15, %r27, %r3;
@@ -9272,7 +11217,7 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
-; SM60-NEXT: @%p18 bra $L__BB196_3;
+; SM60-NEXT: @%p18 bra $L__BB262_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
@@ -9298,7 +11243,7 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
; SM60-NEXT: setp.eq.b32 %p3, %r5, 0;
; SM60-NEXT: setp.eq.b32 %p7, %r6, 0;
-; SM60-NEXT: $L__BB197_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB263_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p1, %r49, %r5;
; SM60-NEXT: max.f32 %r9, %r49, %r5;
@@ -9328,14 +11273,14 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB197_1;
+; SM60-NEXT: @%p9 bra $L__BB263_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r51, %rd12;
; SM60-NEXT: setp.eq.b32 %p12, %r7, 0;
; SM60-NEXT: setp.eq.b32 %p16, %r8, 0;
-; SM60-NEXT: $L__BB197_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB263_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p10, %r51, %r7;
; SM60-NEXT: max.f32 %r19, %r51, %r7;
@@ -9365,14 +11310,14 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r51, %rd21;
-; SM60-NEXT: @%p18 bra $L__BB197_3;
+; SM60-NEXT: @%p18 bra $L__BB263_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
; SM60-NEXT: cvt.u32.u64 %r53, %rd22;
; SM60-NEXT: setp.eq.b32 %p21, %r1, 0;
; SM60-NEXT: setp.eq.b32 %p25, %r2, 0;
-; SM60-NEXT: $L__BB197_5: // %atomicrmw.start15
+; SM60-NEXT: $L__BB263_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p19, %r53, %r1;
; SM60-NEXT: max.f32 %r29, %r53, %r1;
@@ -9402,14 +11347,14 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
; SM60-NEXT: cvt.u32.u64 %r53, %rd31;
-; SM60-NEXT: @%p27 bra $L__BB197_5;
+; SM60-NEXT: @%p27 bra $L__BB263_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
; SM60-NEXT: cvt.u32.u64 %r55, %rd32;
; SM60-NEXT: setp.eq.b32 %p30, %r3, 0;
; SM60-NEXT: setp.eq.b32 %p34, %r4, 0;
-; SM60-NEXT: $L__BB197_7: // %atomicrmw.start20
+; SM60-NEXT: $L__BB263_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f32 %p28, %r55, %r3;
; SM60-NEXT: max.f32 %r39, %r55, %r3;
@@ -9439,7 +11384,7 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
; SM60-NEXT: cvt.u32.u64 %r55, %rd41;
-; SM60-NEXT: @%p36 bra $L__BB197_7;
+; SM60-NEXT: @%p36 bra $L__BB263_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
@@ -9449,6 +11394,23 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x float> %retval
}
+define <1 x double> @fadd_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM60-LABEL: fadd_acq_rel_v1f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fadd_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM60-LABEL: fadd_acq_rel_v2f64_global_cta(
; SM60: {
@@ -9519,6 +11481,32 @@ define <8 x double> @fadd_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x double> %retval
}
+define <1 x double> @fsub_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM60-LABEL: fsub_acq_rel_v1f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [fsub_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd5, [%rd3];
+; SM60-NEXT: $L__BB268_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd4, %rd5, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM60-NEXT: mov.b64 %rd5, %rd2;
+; SM60-NEXT: @%p1 bra $L__BB268_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM60-LABEL: fsub_acq_rel_v2f64_global_cta(
; SM60: {
@@ -9530,22 +11518,22 @@ define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v2f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd5];
-; SM60-NEXT: $L__BB201_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB269_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f64 %rd6, %rd8, %rd1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
; SM60-NEXT: mov.b64 %rd8, %rd3;
-; SM60-NEXT: @%p1 bra $L__BB201_1;
+; SM60-NEXT: @%p1 bra $L__BB269_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd5+8];
-; SM60-NEXT: $L__BB201_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB269_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f64 %rd7, %rd9, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
; SM60-NEXT: mov.b64 %rd9, %rd4;
-; SM60-NEXT: @%p2 bra $L__BB201_3;
+; SM60-NEXT: @%p2 bra $L__BB269_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -9566,40 +11554,40 @@ define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v4f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd9];
-; SM60-NEXT: $L__BB202_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB270_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd5, %rd14;
; SM60-NEXT: sub.rn.f64 %rd10, %rd5, %rd3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
; SM60-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
-; SM60-NEXT: @%p1 bra $L__BB202_1;
+; SM60-NEXT: @%p1 bra $L__BB270_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd9+8];
-; SM60-NEXT: $L__BB202_3: // %atomicrmw.start5
+; SM60-NEXT: $L__BB270_3: // %atomicrmw.start5
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd6, %rd15;
; SM60-NEXT: sub.rn.f64 %rd11, %rd6, %rd4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
; SM60-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
-; SM60-NEXT: @%p2 bra $L__BB202_3;
+; SM60-NEXT: @%p2 bra $L__BB270_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end4
; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd9+16];
-; SM60-NEXT: $L__BB202_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB270_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f64 %rd12, %rd16, %rd1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
; SM60-NEXT: mov.b64 %rd16, %rd7;
-; SM60-NEXT: @%p3 bra $L__BB202_5;
+; SM60-NEXT: @%p3 bra $L__BB270_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd9+24];
-; SM60-NEXT: $L__BB202_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB270_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f64 %rd13, %rd17, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
; SM60-NEXT: mov.b64 %rd17, %rd8;
-; SM60-NEXT: @%p4 bra $L__BB202_7;
+; SM60-NEXT: @%p4 bra $L__BB270_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -9623,76 +11611,76 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fsub_acq_rel_v8f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd17];
-; SM60-NEXT: $L__BB203_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB271_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd7, %rd26;
; SM60-NEXT: sub.rn.f64 %rd18, %rd7, %rd5;
; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
; SM60-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
-; SM60-NEXT: @%p1 bra $L__BB203_1;
+; SM60-NEXT: @%p1 bra $L__BB271_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd17+8];
-; SM60-NEXT: $L__BB203_3: // %atomicrmw.start9
+; SM60-NEXT: $L__BB271_3: // %atomicrmw.start9
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd8, %rd27;
; SM60-NEXT: sub.rn.f64 %rd19, %rd8, %rd6;
; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
; SM60-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
-; SM60-NEXT: @%p2 bra $L__BB203_3;
+; SM60-NEXT: @%p2 bra $L__BB271_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end8
; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd17+16];
-; SM60-NEXT: $L__BB203_5: // %atomicrmw.start20
+; SM60-NEXT: $L__BB271_5: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd9, %rd28;
; SM60-NEXT: sub.rn.f64 %rd20, %rd9, %rd3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
; SM60-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
-; SM60-NEXT: @%p3 bra $L__BB203_5;
+; SM60-NEXT: @%p3 bra $L__BB271_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end19
; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd17+24];
-; SM60-NEXT: $L__BB203_7: // %atomicrmw.start26
+; SM60-NEXT: $L__BB271_7: // %atomicrmw.start26
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd10, %rd29;
; SM60-NEXT: sub.rn.f64 %rd21, %rd10, %rd4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
; SM60-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
-; SM60-NEXT: @%p4 bra $L__BB203_7;
+; SM60-NEXT: @%p4 bra $L__BB271_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end25
; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd17+32];
-; SM60-NEXT: $L__BB203_9: // %atomicrmw.start41
+; SM60-NEXT: $L__BB271_9: // %atomicrmw.start41
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd13, %rd30;
; SM60-NEXT: sub.rn.f64 %rd22, %rd13, %rd1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
-; SM60-NEXT: @%p5 bra $L__BB203_9;
+; SM60-NEXT: @%p5 bra $L__BB271_9;
; SM60-NEXT: // %bb.10: // %atomicrmw.end40
; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd17+40];
-; SM60-NEXT: $L__BB203_11: // %atomicrmw.start47
+; SM60-NEXT: $L__BB271_11: // %atomicrmw.start47
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd14, %rd31;
; SM60-NEXT: sub.rn.f64 %rd23, %rd14, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
; SM60-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
-; SM60-NEXT: @%p6 bra $L__BB203_11;
+; SM60-NEXT: @%p6 bra $L__BB271_11;
; SM60-NEXT: // %bb.12: // %atomicrmw.end46
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd17+48];
-; SM60-NEXT: $L__BB203_13: // %atomicrmw.start58
+; SM60-NEXT: $L__BB271_13: // %atomicrmw.start58
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f64 %rd24, %rd32, %rd11;
; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
; SM60-NEXT: mov.b64 %rd32, %rd15;
-; SM60-NEXT: @%p7 bra $L__BB203_13;
+; SM60-NEXT: @%p7 bra $L__BB271_13;
; SM60-NEXT: // %bb.14: // %atomicrmw.end57
; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd17+56];
-; SM60-NEXT: $L__BB203_15: // %atomicrmw.start64
+; SM60-NEXT: $L__BB271_15: // %atomicrmw.start64
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f64 %rd25, %rd33, %rd12;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
; SM60-NEXT: mov.b64 %rd33, %rd16;
-; SM60-NEXT: @%p8 bra $L__BB203_15;
+; SM60-NEXT: @%p8 bra $L__BB271_15;
; SM60-NEXT: // %bb.16: // %atomicrmw.end63
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -9704,6 +11692,32 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x double> %retval
}
+define <1 x double> @fmin_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM60-LABEL: fmin_acq_rel_v1f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd5, [%rd3];
+; SM60-NEXT: $L__BB272_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd4, %rd5, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM60-NEXT: mov.b64 %rd5, %rd2;
+; SM60-NEXT: @%p1 bra $L__BB272_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM60-LABEL: fmin_acq_rel_v2f64_global_cta(
; SM60: {
@@ -9715,22 +11729,22 @@ define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v2f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd5];
-; SM60-NEXT: $L__BB204_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB273_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: min.f64 %rd6, %rd8, %rd1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
; SM60-NEXT: mov.b64 %rd8, %rd3;
-; SM60-NEXT: @%p1 bra $L__BB204_1;
+; SM60-NEXT: @%p1 bra $L__BB273_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd5+8];
-; SM60-NEXT: $L__BB204_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB273_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: min.f64 %rd7, %rd9, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
; SM60-NEXT: mov.b64 %rd9, %rd4;
-; SM60-NEXT: @%p2 bra $L__BB204_3;
+; SM60-NEXT: @%p2 bra $L__BB273_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -9751,40 +11765,40 @@ define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v4f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd9];
-; SM60-NEXT: $L__BB205_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB274_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd5, %rd14;
; SM60-NEXT: min.f64 %rd10, %rd5, %rd3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
; SM60-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
-; SM60-NEXT: @%p1 bra $L__BB205_1;
+; SM60-NEXT: @%p1 bra $L__BB274_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd9+8];
-; SM60-NEXT: $L__BB205_3: // %atomicrmw.start5
+; SM60-NEXT: $L__BB274_3: // %atomicrmw.start5
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd6, %rd15;
; SM60-NEXT: min.f64 %rd11, %rd6, %rd4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
; SM60-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
-; SM60-NEXT: @%p2 bra $L__BB205_3;
+; SM60-NEXT: @%p2 bra $L__BB274_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end4
; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd9+16];
-; SM60-NEXT: $L__BB205_5: // %atomicrmw.start15
+; SM60-NEXT: $L__BB274_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: min.f64 %rd12, %rd16, %rd1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
; SM60-NEXT: mov.b64 %rd16, %rd7;
-; SM60-NEXT: @%p3 bra $L__BB205_5;
+; SM60-NEXT: @%p3 bra $L__BB274_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd9+24];
-; SM60-NEXT: $L__BB205_7: // %atomicrmw.start20
+; SM60-NEXT: $L__BB274_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: min.f64 %rd13, %rd17, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
; SM60-NEXT: mov.b64 %rd17, %rd8;
-; SM60-NEXT: @%p4 bra $L__BB205_7;
+; SM60-NEXT: @%p4 bra $L__BB274_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -9808,76 +11822,76 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmin_acq_rel_v8f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd17];
-; SM60-NEXT: $L__BB206_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB275_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd7, %rd26;
; SM60-NEXT: min.f64 %rd18, %rd7, %rd5;
; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
; SM60-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
-; SM60-NEXT: @%p1 bra $L__BB206_1;
+; SM60-NEXT: @%p1 bra $L__BB275_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd17+8];
-; SM60-NEXT: $L__BB206_3: // %atomicrmw.start9
+; SM60-NEXT: $L__BB275_3: // %atomicrmw.start9
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd8, %rd27;
; SM60-NEXT: min.f64 %rd19, %rd8, %rd6;
; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
; SM60-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
-; SM60-NEXT: @%p2 bra $L__BB206_3;
+; SM60-NEXT: @%p2 bra $L__BB275_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end8
; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd17+16];
-; SM60-NEXT: $L__BB206_5: // %atomicrmw.start19
+; SM60-NEXT: $L__BB275_5: // %atomicrmw.start19
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd9, %rd28;
; SM60-NEXT: min.f64 %rd20, %rd9, %rd3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
; SM60-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
-; SM60-NEXT: @%p3 bra $L__BB206_5;
+; SM60-NEXT: @%p3 bra $L__BB275_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end18
; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd17+24];
-; SM60-NEXT: $L__BB206_7: // %atomicrmw.start24
+; SM60-NEXT: $L__BB275_7: // %atomicrmw.start24
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd10, %rd29;
; SM60-NEXT: min.f64 %rd21, %rd10, %rd4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
; SM60-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
-; SM60-NEXT: @%p4 bra $L__BB206_7;
+; SM60-NEXT: @%p4 bra $L__BB275_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end23
; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd17+32];
-; SM60-NEXT: $L__BB206_9: // %atomicrmw.start38
+; SM60-NEXT: $L__BB275_9: // %atomicrmw.start38
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd13, %rd30;
; SM60-NEXT: min.f64 %rd22, %rd13, %rd1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
-; SM60-NEXT: @%p5 bra $L__BB206_9;
+; SM60-NEXT: @%p5 bra $L__BB275_9;
; SM60-NEXT: // %bb.10: // %atomicrmw.end37
; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd17+40];
-; SM60-NEXT: $L__BB206_11: // %atomicrmw.start43
+; SM60-NEXT: $L__BB275_11: // %atomicrmw.start43
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd14, %rd31;
; SM60-NEXT: min.f64 %rd23, %rd14, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
; SM60-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
-; SM60-NEXT: @%p6 bra $L__BB206_11;
+; SM60-NEXT: @%p6 bra $L__BB275_11;
; SM60-NEXT: // %bb.12: // %atomicrmw.end42
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd17+48];
-; SM60-NEXT: $L__BB206_13: // %atomicrmw.start53
+; SM60-NEXT: $L__BB275_13: // %atomicrmw.start53
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: min.f64 %rd24, %rd32, %rd11;
; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
; SM60-NEXT: mov.b64 %rd32, %rd15;
-; SM60-NEXT: @%p7 bra $L__BB206_13;
+; SM60-NEXT: @%p7 bra $L__BB275_13;
; SM60-NEXT: // %bb.14: // %atomicrmw.end52
; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd17+56];
-; SM60-NEXT: $L__BB206_15: // %atomicrmw.start58
+; SM60-NEXT: $L__BB275_15: // %atomicrmw.start58
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: min.f64 %rd25, %rd33, %rd12;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
; SM60-NEXT: mov.b64 %rd33, %rd16;
-; SM60-NEXT: @%p8 bra $L__BB206_15;
+; SM60-NEXT: @%p8 bra $L__BB275_15;
; SM60-NEXT: // %bb.16: // %atomicrmw.end57
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -9889,33 +11903,59 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x double> %retval
}
-define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
-; SM60-LABEL: fmax_acq_rel_v2f64_global_cta(
+define <1 x double> @fmax_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM60-LABEL: fmax_acq_rel_v1f64_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<3>;
-; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd5, [%rd3];
+; SM60-NEXT: $L__BB276_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd4, %rd5, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM60-NEXT: mov.b64 %rd5, %rd2;
+; SM60-NEXT: @%p1 bra $L__BB276_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM60-LABEL: fmax_acq_rel_v2f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<10>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd5, [fmax_acq_rel_v2f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v2f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd5];
-; SM60-NEXT: $L__BB207_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB277_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.f64 %rd6, %rd8, %rd1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
; SM60-NEXT: mov.b64 %rd8, %rd3;
-; SM60-NEXT: @%p1 bra $L__BB207_1;
+; SM60-NEXT: @%p1 bra $L__BB277_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd5+8];
-; SM60-NEXT: $L__BB207_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB277_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.f64 %rd7, %rd9, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
; SM60-NEXT: mov.b64 %rd9, %rd4;
-; SM60-NEXT: @%p2 bra $L__BB207_3;
+; SM60-NEXT: @%p2 bra $L__BB277_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -9936,40 +11976,40 @@ define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v4f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd9];
-; SM60-NEXT: $L__BB208_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB278_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd5, %rd14;
; SM60-NEXT: max.f64 %rd10, %rd5, %rd3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
; SM60-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
-; SM60-NEXT: @%p1 bra $L__BB208_1;
+; SM60-NEXT: @%p1 bra $L__BB278_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd9+8];
-; SM60-NEXT: $L__BB208_3: // %atomicrmw.start5
+; SM60-NEXT: $L__BB278_3: // %atomicrmw.start5
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd6, %rd15;
; SM60-NEXT: max.f64 %rd11, %rd6, %rd4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
; SM60-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
-; SM60-NEXT: @%p2 bra $L__BB208_3;
+; SM60-NEXT: @%p2 bra $L__BB278_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end4
; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd9+16];
-; SM60-NEXT: $L__BB208_5: // %atomicrmw.start15
+; SM60-NEXT: $L__BB278_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.f64 %rd12, %rd16, %rd1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
; SM60-NEXT: mov.b64 %rd16, %rd7;
-; SM60-NEXT: @%p3 bra $L__BB208_5;
+; SM60-NEXT: @%p3 bra $L__BB278_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd9+24];
-; SM60-NEXT: $L__BB208_7: // %atomicrmw.start20
+; SM60-NEXT: $L__BB278_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.f64 %rd13, %rd17, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
; SM60-NEXT: mov.b64 %rd17, %rd8;
-; SM60-NEXT: @%p4 bra $L__BB208_7;
+; SM60-NEXT: @%p4 bra $L__BB278_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -9993,76 +12033,76 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmax_acq_rel_v8f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd17];
-; SM60-NEXT: $L__BB209_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB279_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd7, %rd26;
; SM60-NEXT: max.f64 %rd18, %rd7, %rd5;
; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
; SM60-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
-; SM60-NEXT: @%p1 bra $L__BB209_1;
+; SM60-NEXT: @%p1 bra $L__BB279_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd17+8];
-; SM60-NEXT: $L__BB209_3: // %atomicrmw.start9
+; SM60-NEXT: $L__BB279_3: // %atomicrmw.start9
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd8, %rd27;
; SM60-NEXT: max.f64 %rd19, %rd8, %rd6;
; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
; SM60-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
-; SM60-NEXT: @%p2 bra $L__BB209_3;
+; SM60-NEXT: @%p2 bra $L__BB279_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end8
; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd17+16];
-; SM60-NEXT: $L__BB209_5: // %atomicrmw.start19
+; SM60-NEXT: $L__BB279_5: // %atomicrmw.start19
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd9, %rd28;
; SM60-NEXT: max.f64 %rd20, %rd9, %rd3;
; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
; SM60-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
-; SM60-NEXT: @%p3 bra $L__BB209_5;
+; SM60-NEXT: @%p3 bra $L__BB279_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end18
; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd17+24];
-; SM60-NEXT: $L__BB209_7: // %atomicrmw.start24
+; SM60-NEXT: $L__BB279_7: // %atomicrmw.start24
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd10, %rd29;
; SM60-NEXT: max.f64 %rd21, %rd10, %rd4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
; SM60-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
-; SM60-NEXT: @%p4 bra $L__BB209_7;
+; SM60-NEXT: @%p4 bra $L__BB279_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end23
; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd17+32];
-; SM60-NEXT: $L__BB209_9: // %atomicrmw.start38
+; SM60-NEXT: $L__BB279_9: // %atomicrmw.start38
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd13, %rd30;
; SM60-NEXT: max.f64 %rd22, %rd13, %rd1;
; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
-; SM60-NEXT: @%p5 bra $L__BB209_9;
+; SM60-NEXT: @%p5 bra $L__BB279_9;
; SM60-NEXT: // %bb.10: // %atomicrmw.end37
; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd17+40];
-; SM60-NEXT: $L__BB209_11: // %atomicrmw.start43
+; SM60-NEXT: $L__BB279_11: // %atomicrmw.start43
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd14, %rd31;
; SM60-NEXT: max.f64 %rd23, %rd14, %rd2;
; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
; SM60-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
-; SM60-NEXT: @%p6 bra $L__BB209_11;
+; SM60-NEXT: @%p6 bra $L__BB279_11;
; SM60-NEXT: // %bb.12: // %atomicrmw.end42
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd17+48];
-; SM60-NEXT: $L__BB209_13: // %atomicrmw.start53
+; SM60-NEXT: $L__BB279_13: // %atomicrmw.start53
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.f64 %rd24, %rd32, %rd11;
; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
; SM60-NEXT: mov.b64 %rd32, %rd15;
-; SM60-NEXT: @%p7 bra $L__BB209_13;
+; SM60-NEXT: @%p7 bra $L__BB279_13;
; SM60-NEXT: // %bb.14: // %atomicrmw.end52
; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd17+56];
-; SM60-NEXT: $L__BB209_15: // %atomicrmw.start58
+; SM60-NEXT: $L__BB279_15: // %atomicrmw.start58
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: max.f64 %rd25, %rd33, %rd12;
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
; SM60-NEXT: mov.b64 %rd33, %rd16;
-; SM60-NEXT: @%p8 bra $L__BB209_15;
+; SM60-NEXT: @%p8 bra $L__BB279_15;
; SM60-NEXT: // %bb.16: // %atomicrmw.end57
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -10074,6 +12114,40 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x double> %retval
}
+define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM60-LABEL: fminimum_acq_rel_v1f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd3];
+; SM60-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM60-NEXT: $L__BB280_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
+; SM60-NEXT: min.f64 %rd4, %rd9, %rd1;
+; SM60-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd9, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
+; SM60-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
+; SM60-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
+; SM60-NEXT: mov.b64 %rd9, %rd2;
+; SM60-NEXT: @%p5 bra $L__BB280_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM60-LABEL: fminimum_acq_rel_v2f64_global_cta(
; SM60: {
@@ -10086,7 +12160,7 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v2f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd5];
; SM60-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
-; SM60-NEXT: $L__BB210_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB281_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
; SM60-NEXT: min.f64 %rd6, %rd16, %rd1;
@@ -10099,11 +12173,11 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
; SM60-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
; SM60-NEXT: mov.b64 %rd16, %rd3;
-; SM60-NEXT: @%p5 bra $L__BB210_1;
+; SM60-NEXT: @%p5 bra $L__BB281_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd5+8];
; SM60-NEXT: setp.eq.b64 %p8, %rd2, -9223372036854775808;
-; SM60-NEXT: $L__BB210_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB281_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
; SM60-NEXT: min.f64 %rd11, %rd17, %rd2;
@@ -10116,7 +12190,7 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
; SM60-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
; SM60-NEXT: mov.b64 %rd17, %rd4;
-; SM60-NEXT: @%p10 bra $L__BB210_3;
+; SM60-NEXT: @%p10 bra $L__BB281_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -10138,7 +12212,7 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd9];
; SM60-NEXT: setp.eq.b64 %p3, %rd3, -9223372036854775808;
-; SM60-NEXT: $L__BB211_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB282_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd5, %rd30;
; SM60-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
@@ -10151,11 +12225,11 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
-; SM60-NEXT: @%p5 bra $L__BB211_1;
+; SM60-NEXT: @%p5 bra $L__BB282_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd9+8];
; SM60-NEXT: setp.eq.b64 %p8, %rd4, -9223372036854775808;
-; SM60-NEXT: $L__BB211_3: // %atomicrmw.start5
+; SM60-NEXT: $L__BB282_3: // %atomicrmw.start5
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd6, %rd31;
; SM60-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
@@ -10168,11 +12242,11 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
; SM60-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
-; SM60-NEXT: @%p10 bra $L__BB211_3;
+; SM60-NEXT: @%p10 bra $L__BB282_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end4
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd9+16];
; SM60-NEXT: setp.eq.b64 %p13, %rd1, -9223372036854775808;
-; SM60-NEXT: $L__BB211_5: // %atomicrmw.start15
+; SM60-NEXT: $L__BB282_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
; SM60-NEXT: min.f64 %rd20, %rd32, %rd1;
@@ -10185,11 +12259,11 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
; SM60-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
; SM60-NEXT: mov.b64 %rd32, %rd7;
-; SM60-NEXT: @%p15 bra $L__BB211_5;
+; SM60-NEXT: @%p15 bra $L__BB282_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd9+24];
; SM60-NEXT: setp.eq.b64 %p18, %rd2, -9223372036854775808;
-; SM60-NEXT: $L__BB211_7: // %atomicrmw.start20
+; SM60-NEXT: $L__BB282_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
; SM60-NEXT: min.f64 %rd25, %rd33, %rd2;
@@ -10202,7 +12276,7 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
; SM60-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
; SM60-NEXT: mov.b64 %rd33, %rd8;
-; SM60-NEXT: @%p20 bra $L__BB211_7;
+; SM60-NEXT: @%p20 bra $L__BB282_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -10227,7 +12301,7 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd58, [%rd17];
; SM60-NEXT: setp.eq.b64 %p3, %rd5, -9223372036854775808;
-; SM60-NEXT: $L__BB212_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB283_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd7, %rd58;
; SM60-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
@@ -10240,11 +12314,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
; SM60-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
-; SM60-NEXT: @%p5 bra $L__BB212_1;
+; SM60-NEXT: @%p5 bra $L__BB283_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd59, [%rd17+8];
; SM60-NEXT: setp.eq.b64 %p8, %rd6, -9223372036854775808;
-; SM60-NEXT: $L__BB212_3: // %atomicrmw.start9
+; SM60-NEXT: $L__BB283_3: // %atomicrmw.start9
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd8, %rd59;
; SM60-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
@@ -10257,11 +12331,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
; SM60-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
-; SM60-NEXT: @%p10 bra $L__BB212_3;
+; SM60-NEXT: @%p10 bra $L__BB283_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end8
; SM60-NEXT: ld.volatile.global.b64 %rd60, [%rd17+16];
; SM60-NEXT: setp.eq.b64 %p13, %rd3, -9223372036854775808;
-; SM60-NEXT: $L__BB212_5: // %atomicrmw.start19
+; SM60-NEXT: $L__BB283_5: // %atomicrmw.start19
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd9, %rd60;
; SM60-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
@@ -10274,11 +12348,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
; SM60-NEXT: atom.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
; SM60-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
-; SM60-NEXT: @%p15 bra $L__BB212_5;
+; SM60-NEXT: @%p15 bra $L__BB283_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end18
; SM60-NEXT: ld.volatile.global.b64 %rd61, [%rd17+24];
; SM60-NEXT: setp.eq.b64 %p18, %rd4, -9223372036854775808;
-; SM60-NEXT: $L__BB212_7: // %atomicrmw.start24
+; SM60-NEXT: $L__BB283_7: // %atomicrmw.start24
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd10, %rd61;
; SM60-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
@@ -10291,11 +12365,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
; SM60-NEXT: atom.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
; SM60-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
-; SM60-NEXT: @%p20 bra $L__BB212_7;
+; SM60-NEXT: @%p20 bra $L__BB283_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end23
; SM60-NEXT: ld.volatile.global.b64 %rd62, [%rd17+32];
; SM60-NEXT: setp.eq.b64 %p23, %rd1, -9223372036854775808;
-; SM60-NEXT: $L__BB212_9: // %atomicrmw.start38
+; SM60-NEXT: $L__BB283_9: // %atomicrmw.start38
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd13, %rd62;
; SM60-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
@@ -10308,11 +12382,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
; SM60-NEXT: atom.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
; SM60-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
-; SM60-NEXT: @%p25 bra $L__BB212_9;
+; SM60-NEXT: @%p25 bra $L__BB283_9;
; SM60-NEXT: // %bb.10: // %atomicrmw.end37
; SM60-NEXT: ld.volatile.global.b64 %rd63, [%rd17+40];
; SM60-NEXT: setp.eq.b64 %p28, %rd2, -9223372036854775808;
-; SM60-NEXT: $L__BB212_11: // %atomicrmw.start43
+; SM60-NEXT: $L__BB283_11: // %atomicrmw.start43
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd14, %rd63;
; SM60-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
@@ -10325,11 +12399,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
; SM60-NEXT: atom.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
; SM60-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
-; SM60-NEXT: @%p30 bra $L__BB212_11;
+; SM60-NEXT: @%p30 bra $L__BB283_11;
; SM60-NEXT: // %bb.12: // %atomicrmw.end42
; SM60-NEXT: ld.volatile.global.b64 %rd64, [%rd17+48];
; SM60-NEXT: setp.eq.b64 %p33, %rd11, -9223372036854775808;
-; SM60-NEXT: $L__BB212_13: // %atomicrmw.start53
+; SM60-NEXT: $L__BB283_13: // %atomicrmw.start53
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
; SM60-NEXT: min.f64 %rd48, %rd64, %rd11;
@@ -10342,11 +12416,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
; SM60-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
; SM60-NEXT: mov.b64 %rd64, %rd15;
-; SM60-NEXT: @%p35 bra $L__BB212_13;
+; SM60-NEXT: @%p35 bra $L__BB283_13;
; SM60-NEXT: // %bb.14: // %atomicrmw.end52
; SM60-NEXT: ld.volatile.global.b64 %rd65, [%rd17+56];
; SM60-NEXT: setp.eq.b64 %p38, %rd12, -9223372036854775808;
-; SM60-NEXT: $L__BB212_15: // %atomicrmw.start58
+; SM60-NEXT: $L__BB283_15: // %atomicrmw.start58
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
; SM60-NEXT: min.f64 %rd53, %rd65, %rd12;
@@ -10359,7 +12433,7 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
; SM60-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
; SM60-NEXT: mov.b64 %rd65, %rd16;
-; SM60-NEXT: @%p40 bra $L__BB212_15;
+; SM60-NEXT: @%p40 bra $L__BB283_15;
; SM60-NEXT: // %bb.16: // %atomicrmw.end57
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -10371,6 +12445,40 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
ret <8 x double> %retval
}
+define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v1f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd3];
+; SM60-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM60-NEXT: $L__BB284_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
+; SM60-NEXT: max.f64 %rd4, %rd9, %rd1;
+; SM60-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd9, 0;
+; SM60-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
+; SM60-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
+; SM60-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
+; SM60-NEXT: mov.b64 %rd9, %rd2;
+; SM60-NEXT: @%p5 bra $L__BB284_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM60-LABEL: fmaximum_acq_rel_v2f64_global_cta(
; SM60: {
@@ -10383,7 +12491,7 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd5];
; SM60-NEXT: setp.eq.b64 %p3, %rd1, 0;
-; SM60-NEXT: $L__BB213_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB285_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
; SM60-NEXT: max.f64 %rd6, %rd16, %rd1;
@@ -10396,11 +12504,11 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
; SM60-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
; SM60-NEXT: mov.b64 %rd16, %rd3;
-; SM60-NEXT: @%p5 bra $L__BB213_1;
+; SM60-NEXT: @%p5 bra $L__BB285_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd5+8];
; SM60-NEXT: setp.eq.b64 %p8, %rd2, 0;
-; SM60-NEXT: $L__BB213_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB285_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
; SM60-NEXT: max.f64 %rd11, %rd17, %rd2;
@@ -10413,7 +12521,7 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
; SM60-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
; SM60-NEXT: mov.b64 %rd17, %rd4;
-; SM60-NEXT: @%p10 bra $L__BB213_3;
+; SM60-NEXT: @%p10 bra $L__BB285_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -10435,7 +12543,7 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd9];
; SM60-NEXT: setp.eq.b64 %p3, %rd3, 0;
-; SM60-NEXT: $L__BB214_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB286_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd5, %rd30;
; SM60-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
@@ -10448,11 +12556,11 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
-; SM60-NEXT: @%p5 bra $L__BB214_1;
+; SM60-NEXT: @%p5 bra $L__BB286_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd9+8];
; SM60-NEXT: setp.eq.b64 %p8, %rd4, 0;
-; SM60-NEXT: $L__BB214_3: // %atomicrmw.start5
+; SM60-NEXT: $L__BB286_3: // %atomicrmw.start5
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd6, %rd31;
; SM60-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
@@ -10465,11 +12573,11 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
; SM60-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
-; SM60-NEXT: @%p10 bra $L__BB214_3;
+; SM60-NEXT: @%p10 bra $L__BB286_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end4
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd9+16];
; SM60-NEXT: setp.eq.b64 %p13, %rd1, 0;
-; SM60-NEXT: $L__BB214_5: // %atomicrmw.start15
+; SM60-NEXT: $L__BB286_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
; SM60-NEXT: max.f64 %rd20, %rd32, %rd1;
@@ -10482,11 +12590,11 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
; SM60-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
; SM60-NEXT: mov.b64 %rd32, %rd7;
-; SM60-NEXT: @%p15 bra $L__BB214_5;
+; SM60-NEXT: @%p15 bra $L__BB286_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd9+24];
; SM60-NEXT: setp.eq.b64 %p18, %rd2, 0;
-; SM60-NEXT: $L__BB214_7: // %atomicrmw.start20
+; SM60-NEXT: $L__BB286_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
; SM60-NEXT: max.f64 %rd25, %rd33, %rd2;
@@ -10499,7 +12607,7 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
; SM60-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
; SM60-NEXT: mov.b64 %rd33, %rd8;
-; SM60-NEXT: @%p20 bra $L__BB214_7;
+; SM60-NEXT: @%p20 bra $L__BB286_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -10524,7 +12632,7 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd58, [%rd17];
; SM60-NEXT: setp.eq.b64 %p3, %rd5, 0;
-; SM60-NEXT: $L__BB215_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB287_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd7, %rd58;
; SM60-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
@@ -10537,11 +12645,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
; SM60-NEXT: atom.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
; SM60-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
-; SM60-NEXT: @%p5 bra $L__BB215_1;
+; SM60-NEXT: @%p5 bra $L__BB287_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd59, [%rd17+8];
; SM60-NEXT: setp.eq.b64 %p8, %rd6, 0;
-; SM60-NEXT: $L__BB215_3: // %atomicrmw.start9
+; SM60-NEXT: $L__BB287_3: // %atomicrmw.start9
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd8, %rd59;
; SM60-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
@@ -10554,11 +12662,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
; SM60-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
-; SM60-NEXT: @%p10 bra $L__BB215_3;
+; SM60-NEXT: @%p10 bra $L__BB287_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end8
; SM60-NEXT: ld.volatile.global.b64 %rd60, [%rd17+16];
; SM60-NEXT: setp.eq.b64 %p13, %rd3, 0;
-; SM60-NEXT: $L__BB215_5: // %atomicrmw.start19
+; SM60-NEXT: $L__BB287_5: // %atomicrmw.start19
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd9, %rd60;
; SM60-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
@@ -10571,11 +12679,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
; SM60-NEXT: atom.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
; SM60-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
-; SM60-NEXT: @%p15 bra $L__BB215_5;
+; SM60-NEXT: @%p15 bra $L__BB287_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end18
; SM60-NEXT: ld.volatile.global.b64 %rd61, [%rd17+24];
; SM60-NEXT: setp.eq.b64 %p18, %rd4, 0;
-; SM60-NEXT: $L__BB215_7: // %atomicrmw.start24
+; SM60-NEXT: $L__BB287_7: // %atomicrmw.start24
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd10, %rd61;
; SM60-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
@@ -10588,11 +12696,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
; SM60-NEXT: atom.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
; SM60-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
-; SM60-NEXT: @%p20 bra $L__BB215_7;
+; SM60-NEXT: @%p20 bra $L__BB287_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end23
; SM60-NEXT: ld.volatile.global.b64 %rd62, [%rd17+32];
; SM60-NEXT: setp.eq.b64 %p23, %rd1, 0;
-; SM60-NEXT: $L__BB215_9: // %atomicrmw.start38
+; SM60-NEXT: $L__BB287_9: // %atomicrmw.start38
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd13, %rd62;
; SM60-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
@@ -10605,11 +12713,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
; SM60-NEXT: atom.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
; SM60-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
-; SM60-NEXT: @%p25 bra $L__BB215_9;
+; SM60-NEXT: @%p25 bra $L__BB287_9;
; SM60-NEXT: // %bb.10: // %atomicrmw.end37
; SM60-NEXT: ld.volatile.global.b64 %rd63, [%rd17+40];
; SM60-NEXT: setp.eq.b64 %p28, %rd2, 0;
-; SM60-NEXT: $L__BB215_11: // %atomicrmw.start43
+; SM60-NEXT: $L__BB287_11: // %atomicrmw.start43
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b64 %rd14, %rd63;
; SM60-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
@@ -10622,11 +12730,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
; SM60-NEXT: atom.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
; SM60-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
-; SM60-NEXT: @%p30 bra $L__BB215_11;
+; SM60-NEXT: @%p30 bra $L__BB287_11;
; SM60-NEXT: // %bb.12: // %atomicrmw.end42
; SM60-NEXT: ld.volatile.global.b64 %rd64, [%rd17+48];
; SM60-NEXT: setp.eq.b64 %p33, %rd11, 0;
-; SM60-NEXT: $L__BB215_13: // %atomicrmw.start53
+; SM60-NEXT: $L__BB287_13: // %atomicrmw.start53
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
; SM60-NEXT: max.f64 %rd48, %rd64, %rd11;
@@ -10639,11 +12747,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
; SM60-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
; SM60-NEXT: mov.b64 %rd64, %rd15;
-; SM60-NEXT: @%p35 bra $L__BB215_13;
+; SM60-NEXT: @%p35 bra $L__BB287_13;
; SM60-NEXT: // %bb.14: // %atomicrmw.end52
; SM60-NEXT: ld.volatile.global.b64 %rd65, [%rd17+56];
; SM60-NEXT: setp.eq.b64 %p38, %rd12, 0;
-; SM60-NEXT: $L__BB215_15: // %atomicrmw.start58
+; SM60-NEXT: $L__BB287_15: // %atomicrmw.start58
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
; SM60-NEXT: max.f64 %rd53, %rd65, %rd12;
@@ -10656,7 +12764,7 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
; SM60-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
; SM60-NEXT: mov.b64 %rd65, %rd16;
-; SM60-NEXT: @%p40 bra $L__BB215_15;
+; SM60-NEXT: @%p40 bra $L__BB287_15;
; SM60-NEXT: // %bb.16: // %atomicrmw.end57
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -10668,6 +12776,48 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
ret <8 x double> %retval
}
+define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM60-LABEL: fadd_acq_rel_v1f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB288_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: add.rn.f16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB288_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM60-LABEL: fadd_acq_rel_v2f16_global_cta(
; SM60: {
@@ -10680,13 +12830,13 @@ define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NEXT: $L__BB216_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB289_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.rn.f16x2 %r3, %r4, %r2;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB216_1;
+; SM60-NEXT: @%p1 bra $L__BB289_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -10709,7 +12859,7 @@ define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB217_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB290_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.rn.f16x2 %r3, %r5, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -10725,7 +12875,7 @@ define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB217_1;
+; SM60-NEXT: @%p1 bra $L__BB290_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -10748,7 +12898,7 @@ define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB218_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB291_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.rn.f16x2 %r5, %r9, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -10764,12 +12914,12 @@ define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB218_1;
+; SM60-NEXT: @%p1 bra $L__BB291_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: $L__BB218_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB291_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: add.rn.f16x2 %r7, %r11, %r3;
; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -10785,7 +12935,7 @@ define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB218_3;
+; SM60-NEXT: @%p2 bra $L__BB291_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -10794,6 +12944,48 @@ define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
ret <8 x half> %retval
}
+define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM60-LABEL: fsub_acq_rel_v1f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB292_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB292_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM60-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM60: {
@@ -10806,13 +12998,13 @@ define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NEXT: $L__BB219_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB293_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f16x2 %r3, %r4, %r2;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB219_1;
+; SM60-NEXT: @%p1 bra $L__BB293_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -10835,7 +13027,7 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB220_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB294_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f16x2 %r3, %r5, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -10851,7 +13043,7 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB220_1;
+; SM60-NEXT: @%p1 bra $L__BB294_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -10874,7 +13066,7 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB221_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB295_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f16x2 %r5, %r9, %r1;
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -10890,12 +13082,12 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB221_1;
+; SM60-NEXT: @%p1 bra $L__BB295_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: $L__BB221_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB295_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: sub.rn.f16x2 %r7, %r11, %r3;
; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -10911,7 +13103,7 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB221_3;
+; SM60-NEXT: @%p2 bra $L__BB295_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -10920,6 +13112,51 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
ret <8 x half> %retval
}
+define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM60-LABEL: fmin_acq_rel_v1f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<18>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM60-NEXT: $L__BB296_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM60-NEXT: min.f32 %r11, %r9, %r10;
+; SM60-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NEXT: mov.b32 %r17, %r3;
+; SM60-NEXT: @%p1 bra $L__BB296_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM60-LABEL: fmin_acq_rel_v2f16_global_cta(
; SM60: {
@@ -10936,7 +13173,7 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM60-NEXT: cvt.f32.f16 %r3, %rs2;
; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
-; SM60-NEXT: $L__BB222_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB297_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r10;
; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
@@ -10949,7 +13186,7 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r10;
; SM60-NEXT: mov.b32 %r10, %r1;
-; SM60-NEXT: @%p1 bra $L__BB222_1;
+; SM60-NEXT: @%p1 bra $L__BB297_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -10979,7 +13216,7 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
; SM60-NEXT: cvt.f32.f16 %r10, %rs8;
; SM60-NEXT: cvt.f32.f16 %r13, %rs7;
-; SM60-NEXT: $L__BB223_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB298_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r17;
; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
@@ -11009,7 +13246,7 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB223_1;
+; SM60-NEXT: @%p1 bra $L__BB298_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r17, %r18};
@@ -11037,7 +13274,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: cvt.f32.f16 %r5, %rs2;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
; SM60-NEXT: cvt.f32.f16 %r12, %rs8;
-; SM60-NEXT: $L__BB224_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB299_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r33;
; SM60-NEXT: cvt.f32.f16 %r6, %rs4;
@@ -11069,7 +13306,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB224_1;
+; SM60-NEXT: @%p1 bra $L__BB299_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
@@ -11078,7 +13315,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: cvt.f32.f16 %r19, %rs14;
; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
; SM60-NEXT: cvt.f32.f16 %r26, %rs20;
-; SM60-NEXT: $L__BB224_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB299_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r35;
; SM60-NEXT: cvt.f32.f16 %r20, %rs16;
@@ -11110,7 +13347,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r35, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB224_3;
+; SM60-NEXT: @%p2 bra $L__BB299_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
@@ -11119,6 +13356,51 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
ret <8 x half> %retval
}
+define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM60-LABEL: fmax_acq_rel_v1f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<18>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM60-NEXT: $L__BB300_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM60-NEXT: max.f32 %r11, %r9, %r10;
+; SM60-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NEXT: mov.b32 %r17, %r3;
+; SM60-NEXT: @%p1 bra $L__BB300_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM60-LABEL: fmax_acq_rel_v2f16_global_cta(
; SM60: {
@@ -11135,7 +13417,7 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM60-NEXT: cvt.f32.f16 %r3, %rs2;
; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
-; SM60-NEXT: $L__BB225_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB301_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r10;
; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
@@ -11148,7 +13430,7 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r10;
; SM60-NEXT: mov.b32 %r10, %r1;
-; SM60-NEXT: @%p1 bra $L__BB225_1;
+; SM60-NEXT: @%p1 bra $L__BB301_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -11178,7 +13460,7 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
; SM60-NEXT: cvt.f32.f16 %r10, %rs8;
; SM60-NEXT: cvt.f32.f16 %r13, %rs7;
-; SM60-NEXT: $L__BB226_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB302_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r17;
; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
@@ -11208,7 +13490,7 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB226_1;
+; SM60-NEXT: @%p1 bra $L__BB302_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r17, %r18};
@@ -11236,7 +13518,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: cvt.f32.f16 %r5, %rs2;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
; SM60-NEXT: cvt.f32.f16 %r12, %rs8;
-; SM60-NEXT: $L__BB227_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB303_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r33;
; SM60-NEXT: cvt.f32.f16 %r6, %rs4;
@@ -11268,7 +13550,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB227_1;
+; SM60-NEXT: @%p1 bra $L__BB303_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
@@ -11277,7 +13559,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: cvt.f32.f16 %r19, %rs14;
; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
; SM60-NEXT: cvt.f32.f16 %r26, %rs20;
-; SM60-NEXT: $L__BB227_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB303_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r35;
; SM60-NEXT: cvt.f32.f16 %r20, %rs16;
@@ -11309,7 +13591,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r35, %rd21;
-; SM60-NEXT: @%p2 bra $L__BB227_3;
+; SM60-NEXT: @%p2 bra $L__BB303_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
@@ -11318,6 +13600,58 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
ret <8 x half> %retval
}
+define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM60-LABEL: fminimum_acq_rel_v1f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<7>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM60-NEXT: $L__BB304_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: setp.lt.f16 %p1, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-NEXT: mov.b16 %rs7, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
+; SM60-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p6 bra $L__BB304_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM60-LABEL: fminimum_acq_rel_v2f16_global_cta(
; SM60: {
@@ -11334,7 +13668,7 @@ define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
-; SM60-NEXT: $L__BB228_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB305_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM60-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
@@ -11360,7 +13694,7 @@ define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p11, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p11 bra $L__BB228_1;
+; SM60-NEXT: @%p11 bra $L__BB305_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -11390,7 +13724,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
; SM60-NEXT: setp.eq.b16 %p14, %rs17, -32768;
; SM60-NEXT: setp.eq.b16 %p19, %rs16, -32768;
-; SM60-NEXT: $L__BB229_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB306_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
; SM60-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
@@ -11445,7 +13779,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB229_1;
+; SM60-NEXT: @%p21 bra $L__BB306_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -11475,7 +13809,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
; SM60-NEXT: setp.eq.b16 %p14, %rs17, -32768;
; SM60-NEXT: setp.eq.b16 %p19, %rs16, -32768;
-; SM60-NEXT: $L__BB230_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB307_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
; SM60-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
@@ -11530,12 +13864,12 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB230_1;
+; SM60-NEXT: @%p21 bra $L__BB307_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: $L__BB230_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB307_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs30, %rs31}, %r3;
; SM60-NEXT: mov.b32 {%rs32, %rs33}, %r11;
@@ -11596,7 +13930,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p42 bra $L__BB230_3;
+; SM60-NEXT: @%p42 bra $L__BB307_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -11605,6 +13939,58 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x half> %retval
}
+define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v1f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<7>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM60-NEXT: $L__BB308_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: setp.gt.f16 %p1, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-NEXT: mov.b16 %rs7, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
+; SM60-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p6 bra $L__BB308_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM60-LABEL: fmaximum_acq_rel_v2f16_global_cta(
; SM60: {
@@ -11621,7 +14007,7 @@ define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
-; SM60-NEXT: $L__BB231_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB309_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM60-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
@@ -11647,7 +14033,7 @@ define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM60-NEXT: setp.ne.b32 %p11, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p11 bra $L__BB231_1;
+; SM60-NEXT: @%p11 bra $L__BB309_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -11677,7 +14063,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
; SM60-NEXT: setp.eq.b16 %p14, %rs17, 0;
; SM60-NEXT: setp.eq.b16 %p19, %rs16, 0;
-; SM60-NEXT: $L__BB232_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB310_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
; SM60-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
@@ -11732,7 +14118,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB232_1;
+; SM60-NEXT: @%p21 bra $L__BB310_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -11762,7 +14148,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
; SM60-NEXT: setp.eq.b16 %p14, %rs17, 0;
; SM60-NEXT: setp.eq.b16 %p19, %rs16, 0;
-; SM60-NEXT: $L__BB233_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB311_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
; SM60-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
@@ -11817,12 +14203,12 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB233_1;
+; SM60-NEXT: @%p21 bra $L__BB311_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM60-NEXT: $L__BB233_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB311_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs30, %rs31}, %r3;
; SM60-NEXT: mov.b32 {%rs32, %rs33}, %r11;
@@ -11883,7 +14269,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p42 bra $L__BB233_3;
+; SM60-NEXT: @%p42 bra $L__BB311_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -11892,6 +14278,56 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x half> %retval
}
+define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM60-LABEL: fadd_acq_rel_v1bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<2>;
+; SM60-NEXT: .reg .b32 %r<24>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NEXT: $L__BB312_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NEXT: mov.b32 %r23, %r3;
+; SM60-NEXT: @%p2 bra $L__BB312_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM60-LABEL: fadd_acq_rel_v2bf16_global_cta(
; SM60: {
@@ -11910,7 +14346,7 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: shl.b32 %r4, %r3, 16;
; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
; SM60-NEXT: shl.b32 %r14, %r13, 16;
-; SM60-NEXT: $L__BB234_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB313_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r24;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -11935,7 +14371,7 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
; SM60-NEXT: setp.ne.b32 %p3, %r1, %r24;
; SM60-NEXT: mov.b32 %r24, %r1;
-; SM60-NEXT: @%p3 bra $L__BB234_1;
+; SM60-NEXT: @%p3 bra $L__BB313_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -11969,7 +14405,7 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: shl.b32 %r25, %r24, 16;
; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
; SM60-NEXT: shl.b32 %r35, %r34, 16;
-; SM60-NEXT: $L__BB235_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB314_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -12023,7 +14459,7 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r45, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB235_1;
+; SM60-NEXT: @%p5 bra $L__BB314_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
@@ -12053,7 +14489,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
; SM60-NEXT: shl.b32 %r27, %r26, 16;
-; SM60-NEXT: $L__BB236_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB315_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
@@ -12111,14 +14547,14 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r89, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB236_1;
+; SM60-NEXT: @%p5 bra $L__BB315_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r91, %rd12;
; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r4;
-; SM60-NEXT: $L__BB236_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB315_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u32.u16 %r47, %rs10;
; SM60-NEXT: shl.b32 %r48, %r47, 16;
@@ -12180,7 +14616,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r91, %rd21;
-; SM60-NEXT: @%p10 bra $L__BB236_3;
+; SM60-NEXT: @%p10 bra $L__BB315_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
@@ -12189,6 +14625,56 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x bfloat> %retval
}
+define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM60-LABEL: fsub_acq_rel_v1bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<2>;
+; SM60-NEXT: .reg .b32 %r<24>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NEXT: $L__BB316_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: sub.rn.f32 %r12, %r9, %r11;
+; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NEXT: mov.b32 %r23, %r3;
+; SM60-NEXT: @%p2 bra $L__BB316_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM60-LABEL: fsub_acq_rel_v2bf16_global_cta(
; SM60: {
@@ -12207,7 +14693,7 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: shl.b32 %r4, %r3, 16;
; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
; SM60-NEXT: shl.b32 %r14, %r13, 16;
-; SM60-NEXT: $L__BB237_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB317_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r24;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -12232,7 +14718,7 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
; SM60-NEXT: setp.ne.b32 %p3, %r1, %r24;
; SM60-NEXT: mov.b32 %r24, %r1;
-; SM60-NEXT: @%p3 bra $L__BB237_1;
+; SM60-NEXT: @%p3 bra $L__BB317_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -12266,7 +14752,7 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: shl.b32 %r25, %r24, 16;
; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
; SM60-NEXT: shl.b32 %r35, %r34, 16;
-; SM60-NEXT: $L__BB238_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB318_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -12320,7 +14806,7 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r45, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB238_1;
+; SM60-NEXT: @%p5 bra $L__BB318_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
@@ -12350,7 +14836,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
; SM60-NEXT: shl.b32 %r27, %r26, 16;
-; SM60-NEXT: $L__BB239_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB319_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
@@ -12408,14 +14894,14 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r89, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB239_1;
+; SM60-NEXT: @%p5 bra $L__BB319_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r91, %rd12;
; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r4;
-; SM60-NEXT: $L__BB239_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB319_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u32.u16 %r47, %rs10;
; SM60-NEXT: shl.b32 %r48, %r47, 16;
@@ -12477,7 +14963,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r91, %rd21;
-; SM60-NEXT: @%p10 bra $L__BB239_3;
+; SM60-NEXT: @%p10 bra $L__BB319_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
@@ -12486,13 +14972,63 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x bfloat> %retval
}
-define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
-; SM60-LABEL: fmin_acq_rel_v2bf16_global_cta(
+define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM60-LABEL: fmin_acq_rel_v1bf16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<4>;
-; SM60-NEXT: .reg .b16 %rs<5>;
-; SM60-NEXT: .reg .b32 %r<25>;
-; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<2>;
+; SM60-NEXT: .reg .b32 %r<24>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NEXT: $L__BB320_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: min.f32 %r12, %r9, %r11;
+; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NEXT: mov.b32 %r23, %r3;
+; SM60-NEXT: @%p2 bra $L__BB320_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM60-LABEL: fmin_acq_rel_v2bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
@@ -12504,7 +15040,7 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: shl.b32 %r4, %r3, 16;
; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
; SM60-NEXT: shl.b32 %r14, %r13, 16;
-; SM60-NEXT: $L__BB240_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB321_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r24;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -12529,7 +15065,7 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
; SM60-NEXT: setp.ne.b32 %p3, %r1, %r24;
; SM60-NEXT: mov.b32 %r24, %r1;
-; SM60-NEXT: @%p3 bra $L__BB240_1;
+; SM60-NEXT: @%p3 bra $L__BB321_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -12563,7 +15099,7 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: shl.b32 %r25, %r24, 16;
; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
; SM60-NEXT: shl.b32 %r35, %r34, 16;
-; SM60-NEXT: $L__BB241_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB322_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -12617,7 +15153,7 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r45, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB241_1;
+; SM60-NEXT: @%p5 bra $L__BB322_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
@@ -12647,7 +15183,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
; SM60-NEXT: shl.b32 %r27, %r26, 16;
-; SM60-NEXT: $L__BB242_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB323_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
@@ -12705,14 +15241,14 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r89, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB242_1;
+; SM60-NEXT: @%p5 bra $L__BB323_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r91, %rd12;
; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r4;
-; SM60-NEXT: $L__BB242_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB323_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u32.u16 %r47, %rs10;
; SM60-NEXT: shl.b32 %r48, %r47, 16;
@@ -12774,7 +15310,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r91, %rd21;
-; SM60-NEXT: @%p10 bra $L__BB242_3;
+; SM60-NEXT: @%p10 bra $L__BB323_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
@@ -12783,6 +15319,56 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x bfloat> %retval
}
+define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM60-LABEL: fmax_acq_rel_v1bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<2>;
+; SM60-NEXT: .reg .b32 %r<24>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NEXT: $L__BB324_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: max.f32 %r12, %r9, %r11;
+; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NEXT: mov.b32 %r23, %r3;
+; SM60-NEXT: @%p2 bra $L__BB324_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM60-LABEL: fmax_acq_rel_v2bf16_global_cta(
; SM60: {
@@ -12801,7 +15387,7 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: shl.b32 %r4, %r3, 16;
; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
; SM60-NEXT: shl.b32 %r14, %r13, 16;
-; SM60-NEXT: $L__BB243_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB325_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r24;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -12826,7 +15412,7 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
; SM60-NEXT: setp.ne.b32 %p3, %r1, %r24;
; SM60-NEXT: mov.b32 %r24, %r1;
-; SM60-NEXT: @%p3 bra $L__BB243_1;
+; SM60-NEXT: @%p3 bra $L__BB325_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -12860,7 +15446,7 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: shl.b32 %r25, %r24, 16;
; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
; SM60-NEXT: shl.b32 %r35, %r34, 16;
-; SM60-NEXT: $L__BB244_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB326_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -12914,7 +15500,7 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r45, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB244_1;
+; SM60-NEXT: @%p5 bra $L__BB326_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
@@ -12944,7 +15530,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
; SM60-NEXT: shl.b32 %r27, %r26, 16;
-; SM60-NEXT: $L__BB245_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB327_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
@@ -13002,14 +15588,14 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r89, %rd11;
-; SM60-NEXT: @%p5 bra $L__BB245_1;
+; SM60-NEXT: @%p5 bra $L__BB327_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r91, %rd12;
; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r4;
-; SM60-NEXT: $L__BB245_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB327_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u32.u16 %r47, %rs10;
; SM60-NEXT: shl.b32 %r48, %r47, 16;
@@ -13071,7 +15657,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r91, %rd21;
-; SM60-NEXT: @%p10 bra $L__BB245_3;
+; SM60-NEXT: @%p10 bra $L__BB327_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
@@ -13080,6 +15666,62 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x bfloat> %retval
}
+define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM60-LABEL: fminimum_acq_rel_v1bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<7>;
+; SM60-NEXT: .reg .b16 %rs<8>;
+; SM60-NEXT: .reg .b32 %r<20>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r19, [%rd1];
+; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM60-NEXT: $L__BB328_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r19, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: setp.lt.f32 %p1, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM60-NEXT: shl.b32 %r13, %r12, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
+; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM60-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM60-NEXT: shl.b32 %r15, %r14, %r1;
+; SM60-NEXT: and.b32 %r16, %r19, %r2;
+; SM60-NEXT: or.b32 %r17, %r16, %r15;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM60-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM60-NEXT: mov.b32 %r19, %r3;
+; SM60-NEXT: @%p6 bra $L__BB328_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r18, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r18;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM60-LABEL: fminimum_acq_rel_v2bf16_global_cta(
; SM60: {
@@ -13100,7 +15742,7 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
; SM60-NEXT: shl.b32 %r10, %r9, 16;
; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
-; SM60-NEXT: $L__BB246_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB329_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r16;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -13133,7 +15775,7 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
; SM60-NEXT: setp.ne.b32 %p11, %r1, %r16;
; SM60-NEXT: mov.b32 %r16, %r1;
-; SM60-NEXT: @%p11 bra $L__BB246_1;
+; SM60-NEXT: @%p11 bra $L__BB329_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -13171,7 +15813,7 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: cvt.u32.u16 %r22, %rs15;
; SM60-NEXT: shl.b32 %r23, %r22, 16;
; SM60-NEXT: setp.eq.b16 %p19, %rs15, -32768;
-; SM60-NEXT: $L__BB247_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB330_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r29;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -13241,7 +15883,7 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r29, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB247_1;
+; SM60-NEXT: @%p21 bra $L__BB330_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
@@ -13275,7 +15917,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: shl.b32 %r19, %r18, 16;
; SM60-NEXT: setp.eq.b16 %p14, %rs16, -32768;
; SM60-NEXT: setp.eq.b16 %p19, %rs15, -32768;
-; SM60-NEXT: $L__BB248_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB331_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r57;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
@@ -13349,12 +15991,12 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r57, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB248_1;
+; SM60-NEXT: @%p21 bra $L__BB331_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r59, %rd12;
-; SM60-NEXT: $L__BB248_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB331_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs29, %rs30}, %r3;
; SM60-NEXT: cvt.u32.u16 %r31, %rs30;
@@ -13438,7 +16080,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r59, %rd21;
-; SM60-NEXT: @%p42 bra $L__BB248_3;
+; SM60-NEXT: @%p42 bra $L__BB331_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
@@ -13447,6 +16089,62 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
ret <8 x bfloat> %retval
}
+define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v1bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<7>;
+; SM60-NEXT: .reg .b16 %rs<8>;
+; SM60-NEXT: .reg .b32 %r<20>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r19, [%rd1];
+; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM60-NEXT: $L__BB332_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r19, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: setp.gt.f32 %p1, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM60-NEXT: shl.b32 %r13, %r12, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
+; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM60-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM60-NEXT: shl.b32 %r15, %r14, %r1;
+; SM60-NEXT: and.b32 %r16, %r19, %r2;
+; SM60-NEXT: or.b32 %r17, %r16, %r15;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM60-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM60-NEXT: mov.b32 %r19, %r3;
+; SM60-NEXT: @%p6 bra $L__BB332_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r18, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r18;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM60-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
; SM60: {
@@ -13467,7 +16165,7 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
; SM60-NEXT: shl.b32 %r10, %r9, 16;
; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
-; SM60-NEXT: $L__BB249_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB333_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r16;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -13500,7 +16198,7 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
; SM60-NEXT: setp.ne.b32 %p11, %r1, %r16;
; SM60-NEXT: mov.b32 %r16, %r1;
-; SM60-NEXT: @%p11 bra $L__BB249_1;
+; SM60-NEXT: @%p11 bra $L__BB333_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -13538,7 +16236,7 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: cvt.u32.u16 %r22, %rs15;
; SM60-NEXT: shl.b32 %r23, %r22, 16;
; SM60-NEXT: setp.eq.b16 %p19, %rs15, 0;
-; SM60-NEXT: $L__BB250_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB334_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r29;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -13608,7 +16306,7 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r29, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB250_1;
+; SM60-NEXT: @%p21 bra $L__BB334_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
@@ -13642,7 +16340,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: shl.b32 %r19, %r18, 16;
; SM60-NEXT: setp.eq.b16 %p14, %rs16, 0;
; SM60-NEXT: setp.eq.b16 %p19, %rs15, 0;
-; SM60-NEXT: $L__BB251_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB335_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r57;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
@@ -13716,12 +16414,12 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r57, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB251_1;
+; SM60-NEXT: @%p21 bra $L__BB335_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r59, %rd12;
-; SM60-NEXT: $L__BB251_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB335_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs29, %rs30}, %r3;
; SM60-NEXT: cvt.u32.u16 %r31, %rs30;
@@ -13805,7 +16503,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r59, %rd21;
-; SM60-NEXT: @%p42 bra $L__BB251_3;
+; SM60-NEXT: @%p42 bra $L__BB335_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
@@ -13814,151 +16512,45 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
ret <8 x bfloat> %retval
}
-define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM60-LABEL: add_monotonic_v2i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<12>;
-; SM60-NEXT: .reg .b32 %r<16>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_monotonic_v2i8_global_cta_param_1];
-; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM60-NEXT: ld.param.b64 %rd2, [add_monotonic_v2i8_global_cta_param_0];
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: and.b32 %r6, %r5, 3;
-; SM60-NEXT: shl.b32 %r1, %r6, 3;
-; SM60-NEXT: mov.b32 %r7, 65535;
-; SM60-NEXT: shl.b32 %r8, %r7, %r1;
-; SM60-NEXT: not.b32 %r2, %r8;
-; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
-; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM60-NEXT: $L__BB252_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r9, %r15, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
-; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
-; SM60-NEXT: and.b16 %rs9, %rs8, 255;
-; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
-; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
-; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
-; SM60-NEXT: shl.b32 %r11, %r10, %r1;
-; SM60-NEXT: and.b32 %r12, %r15, %r2;
-; SM60-NEXT: or.b32 %r13, %r12, %r11;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
-; SM60-NEXT: mov.b32 %r15, %r3;
-; SM60-NEXT: @%p1 bra $L__BB252_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r14, %r3, %r1;
-; SM60-NEXT: st.param.b16 [func_retval0], %r14;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
- ret <2 x i8> %retval
-}
-
-define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM60-LABEL: add_acquire_v2i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<12>;
-; SM60-NEXT: .reg .b32 %r<16>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
-; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM60-NEXT: ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: and.b32 %r6, %r5, 3;
-; SM60-NEXT: shl.b32 %r1, %r6, 3;
-; SM60-NEXT: mov.b32 %r7, 65535;
-; SM60-NEXT: shl.b32 %r8, %r7, %r1;
-; SM60-NEXT: not.b32 %r2, %r8;
-; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
-; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM60-NEXT: $L__BB253_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r9, %r15, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
-; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
-; SM60-NEXT: and.b16 %rs9, %rs8, 255;
-; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
-; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
-; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
-; SM60-NEXT: shl.b32 %r11, %r10, %r1;
-; SM60-NEXT: and.b32 %r12, %r15, %r2;
-; SM60-NEXT: or.b32 %r13, %r12, %r11;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
-; SM60-NEXT: mov.b32 %r15, %r3;
-; SM60-NEXT: @%p1 bra $L__BB253_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r14, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r14;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
- ret <2 x i8> %retval
-}
-
-define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM60-LABEL: add_release_v2i8_global_cta(
+define <1 x i8> @add_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: add_monotonic_v1i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<12>;
; SM60-NEXT: .reg .b32 %r<16>;
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
-; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM60-NEXT: ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [add_monotonic_v1i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b8 %r6, [add_monotonic_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: and.b32 %r6, %r5, 3;
-; SM60-NEXT: shl.b32 %r1, %r6, 3;
-; SM60-NEXT: mov.b32 %r7, 65535;
-; SM60-NEXT: shl.b32 %r8, %r7, %r1;
-; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
-; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM60-NEXT: $L__BB254_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB336_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r9, %r15, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
-; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
-; SM60-NEXT: and.b16 %rs9, %rs8, 255;
-; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
-; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
-; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
-; SM60-NEXT: shl.b32 %r11, %r10, %r1;
-; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: add.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
; SM60-NEXT: or.b32 %r13, %r12, %r11;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
-; SM60-NEXT: mov.b32 %r15, %r3;
-; SM60-NEXT: @%p1 bra $L__BB254_1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB336_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r14, %r3, %r1;
-; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
- ret <2 x i8> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
+ ret <1 x i8> %retval
}
-define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM60-LABEL: add_seq_cst_v2i8_global_cta(
+define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_monotonic_v2i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
; SM60-NEXT: .reg .b16 %rs<12>;
@@ -13966,10 +16558,9 @@ define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_monotonic_v2i8_global_cta_param_1];
; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM60-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [add_monotonic_v2i8_global_cta_param_0];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
; SM60-NEXT: and.b32 %r6, %r5, 3;
@@ -13979,7 +16570,7 @@ define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM60-NEXT: $L__BB255_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB337_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r15, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -13996,13 +16587,12 @@ define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
; SM60-NEXT: mov.b32 %r15, %r3;
-; SM60-NEXT: @%p1 bra $L__BB255_1;
+; SM60-NEXT: @%p1 bra $L__BB337_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r14, %r3, %r1;
-; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b16 [func_retval0], %r14;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
ret <2 x i8> %retval
}
@@ -14026,7 +16616,7 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: $L__BB256_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB338_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -14050,7 +16640,7 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM60-NEXT: mov.b32 %r18, %r1;
-; SM60-NEXT: @%p1 bra $L__BB256_1;
+; SM60-NEXT: @%p1 bra $L__BB338_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NEXT: ret;
@@ -14058,114 +16648,184 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
ret <4 x i8> %retval
}
-define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM60-LABEL: add_acquire_v4i8_global_cta(
+define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: add_monotonic_v8i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<13>;
-; SM60-NEXT: .reg .b32 %r<19>;
-; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
-; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
-; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: $L__BB257_1: // %atomicrmw.start
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB339_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
-; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
-; SM60-NEXT: mov.b32 %r18, %r1;
-; SM60-NEXT: @%p1 bra $L__BB257_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
- ret <4 x i8> %retval
-}
-
-define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM60-LABEL: add_release_v4i8_global_cta(
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM60-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM60-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM60-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB339_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @add_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: add_acquire_v1i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<13>;
-; SM60-NEXT: .reg .b32 %r<19>;
-; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd2, [add_acquire_v1i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b8 %r6, [add_acquire_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB340_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB340_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
-; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: $L__BB258_1: // %atomicrmw.start
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_acquire_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB341_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
-; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
-; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
-; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
-; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
-; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
-; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
-; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
-; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
-; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
-; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
-; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
-; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
-; SM60-NEXT: mov.b32 %r18, %r1;
-; SM60-NEXT: @%p1 bra $L__BB258_1;
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB341_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
- ret <4 x i8> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
}
-define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM60-LABEL: add_seq_cst_v4i8_global_cta(
+define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: add_acquire_v4i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
; SM60-NEXT: .reg .b16 %rs<13>;
@@ -14173,9 +16833,8 @@ define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
@@ -14185,7 +16844,7 @@ define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: $L__BB259_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB342_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -14209,17 +16868,17 @@ define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM60-NEXT: mov.b32 %r18, %r1;
-; SM60-NEXT: @%p1 bra $L__BB259_1;
+; SM60-NEXT: @%p1 bra $L__BB342_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
ret <4 x i8> %retval
}
-define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM60-LABEL: add_monotonic_v8i8_global_cta(
+define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: add_acquire_v8i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
; SM60-NEXT: .reg .b16 %rs<25>;
@@ -14227,8 +16886,8 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM60-NEXT: .reg .b64 %rd<12>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
@@ -14242,7 +16901,7 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM60-NEXT: $L__BB260_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB343_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -14300,103 +16959,152 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB260_1;
+; SM60-NEXT: @%p1 bra $L__BB343_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
ret <8 x i8> %retval
}
-define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM60-LABEL: add_acquire_v8i8_global_cta(
+define <1 x i8> @add_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: add_release_v1i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<25>;
-; SM60-NEXT: .reg .b32 %r<35>;
-; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: ld.param.b64 %rd2, [add_release_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [add_release_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB344_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB344_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_release_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB345_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB345_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: add_release_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM60-NEXT: $L__BB261_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB346_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
-; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
-; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
-; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
-; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
-; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
-; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
-; SM60-NEXT: add.s16 %rs18, %rs17, %rs16;
-; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
-; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
-; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
-; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
-; SM60-NEXT: add.s16 %rs21, %rs20, %rs19;
-; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
-; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
-; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
-; SM60-NEXT: add.s16 %rs24, %rs23, %rs22;
-; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
-; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
-; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB261_1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB346_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
- ret <8 x i8> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
}
define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
@@ -14424,7 +17132,7 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM60-NEXT: $L__BB262_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB347_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -14482,7 +17190,7 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB262_1;
+; SM60-NEXT: @%p1 bra $L__BB347_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM60-NEXT: ret;
@@ -14490,6 +17198,148 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i8> @add_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: add_seq_cst_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [add_seq_cst_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [add_seq_cst_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB348_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB348_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_seq_cst_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB349_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB349_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: add_seq_cst_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: $L__BB350_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB350_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM60-LABEL: add_seq_cst_v8i8_global_cta(
; SM60: {
@@ -14515,7 +17365,7 @@ define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM60-NEXT: $L__BB263_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB351_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -14573,7 +17423,7 @@ define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB263_1;
+; SM60-NEXT: @%p1 bra $L__BB351_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -14582,6 +17432,22 @@ define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i32> @add_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: add_monotonic_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v1i32_global_cta_param_0];
+; SM60-NEXT: ld.param.b32 %r1, [add_monotonic_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM60-LABEL: add_monotonic_v2i32_global_cta(
; SM60: {
@@ -14599,151 +17465,197 @@ define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i3
ret <2 x i32> %retval
}
-define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM60-LABEL: add_acquire_v2i32_global_cta(
+define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_monotonic_v4i32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b32 %r<9>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
-; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
- ret <2 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
}
-define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM60-LABEL: add_release_v2i32_global_cta(
+define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: add_monotonic_v8i32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b32 %r<17>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @add_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: add_acquire_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v1i32_global_cta_param_0];
+; SM60-NEXT: ld.param.b32 %r1, [add_acquire_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
-; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
- ret <2 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
+ ret <1 x i32> %retval
}
-define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM60-LABEL: add_seq_cst_v2i32_global_cta(
+define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: add_acquire_v2i32_global_cta(
; SM60: {
; SM60-NEXT: .reg .b32 %r<5>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
ret <2 x i32> %retval
}
-define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM60-LABEL: add_monotonic_v4i32_global_cta(
+define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_acquire_v4i32_global_cta(
; SM60: {
; SM60-NEXT: .reg .b32 %r<9>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v4i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
ret <4 x i32> %retval
}
-define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM60-LABEL: add_acquire_v4i32_global_cta(
+define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: add_acquire_v8i32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b32 %r<17>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v4i32_global_cta_param_0];
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v4i32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
-; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
-; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
-; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
- ret <4 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
}
-define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM60-LABEL: add_release_v4i32_global_cta(
+define <1 x i32> @add_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: add_release_v1i32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b32 %r<3>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v1i32_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
-; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
-; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
-; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ld.param.b32 %r1, [add_release_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
- ret <4 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
+ ret <1 x i32> %retval
}
-define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM60-LABEL: add_seq_cst_v4i32_global_cta(
+define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: add_release_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_release_v4i32_global_cta(
; SM60: {
; SM60-NEXT: .reg .b32 %r<9>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
-; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
ret <4 x i32> %retval
}
-define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM60-LABEL: add_monotonic_v8i32_global_cta(
+define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: add_release_v8i32_global_cta(
; SM60: {
; SM60-NEXT: .reg .b32 %r<17>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_release_v8i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
@@ -14755,60 +17667,66 @@ define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i3
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
ret <8 x i32> %retval
}
-define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM60-LABEL: add_acquire_v8i32_global_cta(
+define <1 x i32> @add_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: add_seq_cst_v1i32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b32 %r<3>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
-; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
-; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
-; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
-; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
-; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
-; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
-; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v1i32_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ld.param.b32 %r1, [add_seq_cst_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
- ret <8 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
+ ret <1 x i32> %retval
}
-define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM60-LABEL: add_release_v8i32_global_cta(
+define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: add_seq_cst_v2i32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b32 %r<5>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_release_v8i32_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v8i32_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_release_v8i32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
-; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
-; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
-; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
-; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
-; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
-; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
-; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
-; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
- ret <8 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_seq_cst_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
}
define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
@@ -14838,6 +17756,44 @@ define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: nand_monotonic_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [nand_monotonic_v1i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b8 %r6, [nand_monotonic_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: $L__BB368_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
+; SM60-NEXT: @%p1 bra $L__BB368_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
+; SM60-NEXT: st.param.b8 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: nand_monotonic_v2i8_global_cta(
; SM60: {
@@ -14858,7 +17814,7 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM60-NEXT: shl.b32 %r8, %r7, %r1;
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
-; SM60-NEXT: $L__BB276_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB369_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r18, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -14876,7 +17832,7 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM60-NEXT: mov.b32 %r18, %r3;
-; SM60-NEXT: @%p1 bra $L__BB276_1;
+; SM60-NEXT: @%p1 bra $L__BB369_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r17, %r3, %r1;
; SM60-NEXT: st.param.b16 [func_retval0], %r17;
@@ -14885,6 +17841,110 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
ret <2 x i8> %retval
}
+define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_monotonic_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB370_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB370_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_monotonic_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB371_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB371_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: nand_acquire_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [nand_acquire_v1i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b8 %r6, [nand_acquire_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: $L__BB372_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
+; SM60-NEXT: @%p1 bra $L__BB372_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM60-LABEL: nand_acquire_v2i8_global_cta(
; SM60: {
@@ -14905,7 +17965,7 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: shl.b32 %r8, %r7, %r1;
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
-; SM60-NEXT: $L__BB277_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB373_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r18, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -14923,7 +17983,7 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM60-NEXT: mov.b32 %r18, %r3;
-; SM60-NEXT: @%p1 bra $L__BB277_1;
+; SM60-NEXT: @%p1 bra $L__BB373_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r17, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -14933,56 +17993,114 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
ret <2 x i8> %retval
}
-define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM60-LABEL: nand_release_v2i8_global_cta(
+define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_acquire_v4i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<9>;
-; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB374_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB374_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_acquire_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB375_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB375_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: nand_release_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<17>;
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
-; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM60-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd2, [nand_release_v1i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [nand_release_v1i8_global_cta_param_1];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: and.b32 %r6, %r5, 3;
-; SM60-NEXT: shl.b32 %r1, %r6, 3;
-; SM60-NEXT: mov.b32 %r7, 65535;
-; SM60-NEXT: shl.b32 %r8, %r7, %r1;
-; SM60-NEXT: not.b32 %r2, %r8;
-; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
-; SM60-NEXT: $L__BB278_1: // %atomicrmw.start
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: $L__BB376_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r9, %r18, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM60-NEXT: and.b32 %r11, %r10, %r4;
-; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM60-NEXT: shl.b32 %r14, %r13, %r1;
-; SM60-NEXT: and.b32 %r15, %r18, %r2;
-; SM60-NEXT: or.b32 %r16, %r15, %r14;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM60-NEXT: mov.b32 %r18, %r3;
-; SM60-NEXT: @%p1 bra $L__BB278_1;
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
+; SM60-NEXT: @%p1 bra $L__BB376_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r17, %r3, %r1;
-; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
+; SM60-NEXT: st.param.b8 [func_retval0], %r15;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
- ret <2 x i8> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
+ ret <1 x i8> %retval
}
-define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM60-LABEL: nand_seq_cst_v2i8_global_cta(
+define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: nand_release_v2i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
; SM60-NEXT: .reg .b16 %rs<9>;
@@ -14990,9 +18108,9 @@ define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM60-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -15002,7 +18120,7 @@ define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: shl.b32 %r8, %r7, %r1;
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
-; SM60-NEXT: $L__BB279_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB377_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r18, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -15020,69 +18138,15 @@ define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM60-NEXT: mov.b32 %r18, %r3;
-; SM60-NEXT: @%p1 bra $L__BB279_1;
+; SM60-NEXT: @%p1 bra $L__BB377_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r17, %r3, %r1;
-; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b16 [func_retval0], %r17;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
ret <2 x i8> %retval
}
-define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM60-LABEL: nand_monotonic_v4i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<6>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
-; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB280_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b32 %r3, %r5, %r2;
-; SM60-NEXT: xor.b32 %r4, %r3, -1;
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
-; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
-; SM60-NEXT: mov.b32 %r5, %r1;
-; SM60-NEXT: @%p1 bra $L__BB280_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
- ret <4 x i8> %retval
-}
-
-define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM60-LABEL: nand_acquire_v4i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<6>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
-; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB281_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b32 %r3, %r5, %r2;
-; SM60-NEXT: xor.b32 %r4, %r3, -1;
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
-; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
-; SM60-NEXT: mov.b32 %r5, %r1;
-; SM60-NEXT: @%p1 bra $L__BB281_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
- ret <4 x i8> %retval
-}
-
define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM60-LABEL: nand_release_v4i8_global_cta(
; SM60: {
@@ -15095,14 +18159,14 @@ define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB282_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB378_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r3, %r5, %r2;
; SM60-NEXT: xor.b32 %r4, %r3, -1;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM60-NEXT: mov.b32 %r5, %r1;
-; SM60-NEXT: @%p1 bra $L__BB282_1;
+; SM60-NEXT: @%p1 bra $L__BB378_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NEXT: ret;
@@ -15110,48 +18174,21 @@ define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
ret <4 x i8> %retval
}
-define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM60-LABEL: nand_seq_cst_v4i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<6>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [nand_seq_cst_v4i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB283_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b32 %r3, %r5, %r2;
-; SM60-NEXT: xor.b32 %r4, %r3, -1;
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
-; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
-; SM60-NEXT: mov.b32 %r5, %r1;
-; SM60-NEXT: @%p1 bra $L__BB283_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
- ret <4 x i8> %retval
-}
-
-define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM60-LABEL: nand_monotonic_v8i8_global_cta(
+define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_release_v8i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
; SM60-NEXT: .reg .b32 %r<9>;
; SM60-NEXT: .reg .b64 %rd<12>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: $L__BB284_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB379_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -15169,488 +18206,250 @@ define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB284_1;
+; SM60-NEXT: @%p1 bra $L__BB379_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
ret <8 x i8> %retval
}
-define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM60-LABEL: nand_acquire_v8i8_global_cta(
+define <1 x i8> @nand_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: nand_seq_cst_v1i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<9>;
-; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [nand_seq_cst_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: $L__BB285_1: // %atomicrmw.start
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: $L__BB380_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r3, %r8, %r2;
-; SM60-NEXT: and.b32 %r4, %r7, %r1;
-; SM60-NEXT: xor.b32 %r5, %r4, -1;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM60-NEXT: xor.b32 %r6, %r3, -1;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
-; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB285_1;
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
+; SM60-NEXT: @%p1 bra $L__BB380_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: st.param.b8 [func_retval0], %r15;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
- ret <8 x i8> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
+ ret <1 x i8> %retval
}
-define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM60-LABEL: nand_release_v8i8_global_cta(
+define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: nand_seq_cst_v2i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<9>;
-; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: $L__BB286_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r3, %r8, %r2;
-; SM60-NEXT: and.b32 %r4, %r7, %r1;
-; SM60-NEXT: xor.b32 %r5, %r4, -1;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM60-NEXT: xor.b32 %r6, %r3, -1;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
-; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB286_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
- ret <8 x i8> %retval
-}
-
-define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM60-LABEL: nand_seq_cst_v8i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<9>;
-; SM60-NEXT: .reg .b64 %rd<12>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: $L__BB287_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r3, %r8, %r2;
-; SM60-NEXT: and.b32 %r4, %r7, %r1;
-; SM60-NEXT: xor.b32 %r5, %r4, -1;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM60-NEXT: xor.b32 %r6, %r3, -1;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
-; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB287_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
- ret <8 x i8> %retval
-}
-
-define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM60-LABEL: nand_monotonic_v2i32_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<7>;
-; SM60-NEXT: .reg .b64 %rd<13>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB288_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r3, %r6, %r2;
-; SM60-NEXT: and.b32 %r4, %r5, %r1;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: not.b64 %rd11, %rd10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB288_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
- ret <2 x i32> %retval
-}
-
-define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM60-LABEL: nand_acquire_v2i32_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<7>;
-; SM60-NEXT: .reg .b64 %rd<13>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB289_1: // %atomicrmw.start
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: $L__BB381_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r3, %r6, %r2;
-; SM60-NEXT: and.b32 %r4, %r5, %r1;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: not.b64 %rd11, %rd10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB289_1;
+; SM60-NEXT: shr.u32 %r9, %r18, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, %r4;
+; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM60-NEXT: shl.b32 %r14, %r13, %r1;
+; SM60-NEXT: and.b32 %r15, %r18, %r2;
+; SM60-NEXT: or.b32 %r16, %r15, %r14;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: @%p1 bra $L__BB381_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r17, %r3, %r1;
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
- ret <2 x i32> %retval
-}
-
-define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM60-LABEL: nand_release_v2i32_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<7>;
-; SM60-NEXT: .reg .b64 %rd<13>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB290_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r3, %r6, %r2;
-; SM60-NEXT: and.b32 %r4, %r5, %r1;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: not.b64 %rd11, %rd10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB290_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: st.param.b16 [func_retval0], %r17;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
- ret <2 x i32> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
}
-define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM60-LABEL: nand_seq_cst_v2i32_global_cta(
+define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_seq_cst_v4i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<7>;
-; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB291_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r3, %r6, %r2;
-; SM60-NEXT: and.b32 %r4, %r5, %r1;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: not.b64 %rd11, %rd10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB291_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.param.b32 %r2, [nand_seq_cst_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
- ret <2 x i32> %retval
-}
-
-define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM60-LABEL: nand_monotonic_v4i32_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<3>;
-; SM60-NEXT: .reg .b32 %r<13>;
-; SM60-NEXT: .reg .b64 %rd<24>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB292_1: // %atomicrmw.start
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB382_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r5, %r10, %r2;
-; SM60-NEXT: and.b32 %r6, %r9, %r1;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: not.b64 %rd11, %rd10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB292_1;
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB382_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
-; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM60-NEXT: $L__BB292_3: // %atomicrmw.start2
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
-; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
-; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
-; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
-; SM60-NEXT: and.b32 %r7, %r12, %r4;
-; SM60-NEXT: and.b32 %r8, %r11, %r3;
-; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
-; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
-; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM60-NEXT: not.b64 %rd22, %rd21;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
-; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
-; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB292_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end1
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
- ret <4 x i32> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
}
-define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM60-LABEL: nand_acquire_v4i32_global_cta(
+define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_seq_cst_v8i8_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<3>;
-; SM60-NEXT: .reg .b32 %r<13>;
-; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB293_1: // %atomicrmw.start
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB383_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r5, %r10, %r2;
-; SM60-NEXT: and.b32 %r6, %r9, %r1;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: not.b64 %rd11, %rd10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB293_1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB383_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
-; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM60-NEXT: $L__BB293_3: // %atomicrmw.start2
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
-; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
-; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
-; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
-; SM60-NEXT: and.b32 %r7, %r12, %r4;
-; SM60-NEXT: and.b32 %r8, %r11, %r3;
-; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
-; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
-; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM60-NEXT: not.b64 %rd22, %rd21;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
-; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
-; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB293_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
- ret <4 x i32> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
}
-define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM60-LABEL: nand_release_v4i32_global_cta(
+define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: nand_monotonic_v1i32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<3>;
-; SM60-NEXT: .reg .b32 %r<13>;
-; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
+; SM60-NEXT: ld.param.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB384_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: not.b32 %r4, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p1 bra $L__BB384_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_monotonic_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB294_1: // %atomicrmw.start
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB385_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r5, %r10, %r2;
-; SM60-NEXT: and.b32 %r6, %r9, %r1;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: not.b64 %rd11, %rd10;
; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB294_1;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB385_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
-; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM60-NEXT: $L__BB294_3: // %atomicrmw.start2
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
-; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
-; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
-; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
-; SM60-NEXT: and.b32 %r7, %r12, %r4;
-; SM60-NEXT: and.b32 %r8, %r11, %r3;
-; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
-; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
-; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM60-NEXT: not.b64 %rd22, %rd21;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
-; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
-; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB294_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end1
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
- ret <4 x i32> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
}
-define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM60-LABEL: nand_seq_cst_v4i32_global_cta(
+define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_monotonic_v4i32_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<3>;
; SM60-NEXT: .reg .b32 %r<13>;
; SM60-NEXT: .reg .b64 %rd<24>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB295_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB386_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -15667,12 +18466,12 @@ define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB295_1;
+; SM60-NEXT: @%p1 bra $L__BB386_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM60-NEXT: $L__BB295_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB386_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
@@ -15689,12 +18488,11 @@ define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB295_3;
+; SM60-NEXT: @%p2 bra $L__BB386_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
-; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
ret <4 x i32> %retval
}
@@ -15712,7 +18510,7 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM60-NEXT: $L__BB296_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB387_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -15729,12 +18527,12 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB296_1;
+; SM60-NEXT: @%p1 bra $L__BB387_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM60-NEXT: $L__BB296_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB387_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -15751,12 +18549,12 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB296_3;
+; SM60-NEXT: @%p2 bra $L__BB387_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM60-NEXT: $L__BB296_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB387_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -15773,12 +18571,12 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM60-NEXT: @%p3 bra $L__BB296_5;
+; SM60-NEXT: @%p3 bra $L__BB387_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM60-NEXT: $L__BB296_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB387_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -15795,13 +18593,140 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM60-NEXT: @%p4 bra $L__BB296_7;
+; SM60-NEXT: @%p4 bra $L__BB387_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
- ret <8 x i32> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: nand_acquire_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
+; SM60-NEXT: ld.param.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB388_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: not.b32 %r4, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p1 bra $L__BB388_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_acquire_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB389_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB389_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_acquire_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB390_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r10, %r2;
+; SM60-NEXT: and.b32 %r6, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB390_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM60-NEXT: $L__BB390_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r7, %r12, %r4;
+; SM60-NEXT: and.b32 %r8, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB390_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
}
define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
@@ -15818,7 +18743,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM60-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB391_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -15835,12 +18760,12 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB297_1;
+; SM60-NEXT: @%p1 bra $L__BB391_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM60-NEXT: $L__BB297_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB391_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -15857,12 +18782,12 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB297_3;
+; SM60-NEXT: @%p2 bra $L__BB391_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM60-NEXT: $L__BB297_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB391_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -15879,12 +18804,12 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM60-NEXT: @%p3 bra $L__BB297_5;
+; SM60-NEXT: @%p3 bra $L__BB391_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM60-NEXT: $L__BB297_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB391_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -15901,7 +18826,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM60-NEXT: @%p4 bra $L__BB297_7;
+; SM60-NEXT: @%p4 bra $L__BB391_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -15911,6 +18836,133 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: nand_release_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [nand_release_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB392_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: not.b32 %r4, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p1 bra $L__BB392_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_release_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB393_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB393_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_release_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB394_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r10, %r2;
+; SM60-NEXT: and.b32 %r6, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB394_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM60-NEXT: $L__BB394_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r7, %r12, %r4;
+; SM60-NEXT: and.b32 %r8, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB394_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM60-LABEL: nand_release_v8i32_global_cta(
; SM60: {
@@ -15926,7 +18978,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM60-NEXT: $L__BB298_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB395_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -15943,12 +18995,12 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB298_1;
+; SM60-NEXT: @%p1 bra $L__BB395_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM60-NEXT: $L__BB298_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB395_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -15965,12 +19017,12 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB298_3;
+; SM60-NEXT: @%p2 bra $L__BB395_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM60-NEXT: $L__BB298_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB395_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -15987,12 +19039,12 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM60-NEXT: @%p3 bra $L__BB298_5;
+; SM60-NEXT: @%p3 bra $L__BB395_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM60-NEXT: $L__BB298_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB395_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -16009,7 +19061,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM60-NEXT: @%p4 bra $L__BB298_7;
+; SM60-NEXT: @%p4 bra $L__BB395_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
@@ -16018,6 +19070,136 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i32> @nand_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: nand_seq_cst_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [nand_seq_cst_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB396_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: not.b32 %r4, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p1 bra $L__BB396_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_seq_cst_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB397_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB397_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_seq_cst_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB398_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r10, %r2;
+; SM60-NEXT: and.b32 %r6, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB398_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM60-NEXT: $L__BB398_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r7, %r12, %r4;
+; SM60-NEXT: and.b32 %r8, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB398_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM60-LABEL: nand_seq_cst_v8i32_global_cta(
; SM60: {
@@ -16033,7 +19215,7 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM60-NEXT: $L__BB299_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB399_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -16050,12 +19232,12 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB299_1;
+; SM60-NEXT: @%p1 bra $L__BB399_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM60-NEXT: $L__BB299_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB399_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -16072,12 +19254,12 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB299_3;
+; SM60-NEXT: @%p2 bra $L__BB399_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM60-NEXT: $L__BB299_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB399_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -16094,12 +19276,12 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM60-NEXT: @%p3 bra $L__BB299_5;
+; SM60-NEXT: @%p3 bra $L__BB399_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM60-NEXT: $L__BB299_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB399_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -16116,7 +19298,7 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM60-NEXT: @%p4 bra $L__BB299_7;
+; SM60-NEXT: @%p4 bra $L__BB399_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
index e2f3877911f15..995d6ec844736 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
@@ -2,6 +2,43 @@
; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s --check-prefix=SM70
; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | %ptxas-verify -arch=sm_70 %}
+define <1 x i8> @xchg_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: xchg_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<14>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r5, [xchg_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
+; SM70-NEXT: mov.b32 %r8, 255;
+; SM70-NEXT: shl.b32 %r9, %r8, %r1;
+; SM70-NEXT: not.b32 %r2, %r9;
+; SM70-NEXT: shl.b32 %r3, %r5, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM70-NEXT: $L__BB0_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r13, %r2;
+; SM70-NEXT: or.b32 %r11, %r10, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM70-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM70-NEXT: mov.b32 %r13, %r4;
+; SM70-NEXT: @%p1 bra $L__BB0_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r12, %r4, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r12;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: xchg_acq_rel_v2i8_global_cta(
; SM70: {
@@ -22,14 +59,14 @@ define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: not.b32 %r2, %r9;
; SM70-NEXT: shl.b32 %r3, %r5, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
-; SM70-NEXT: $L__BB0_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB1_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r10, %r13, %r2;
; SM70-NEXT: or.b32 %r11, %r10, %r3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
; SM70-NEXT: setp.ne.b32 %p1, %r4, %r13;
; SM70-NEXT: mov.b32 %r13, %r4;
-; SM70-NEXT: @%p1 bra $L__BB0_1;
+; SM70-NEXT: @%p1 bra $L__BB1_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r12, %r4, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -70,6 +107,43 @@ define <8 x i8> @xchg_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
ret <8 x i8> %retval
}
+define <1 x i16> @xchg_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: xchg_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<14>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
+; SM70-NEXT: mov.b32 %r8, 65535;
+; SM70-NEXT: shl.b32 %r9, %r8, %r1;
+; SM70-NEXT: not.b32 %r2, %r9;
+; SM70-NEXT: shl.b32 %r3, %r5, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM70-NEXT: $L__BB4_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r13, %r2;
+; SM70-NEXT: or.b32 %r11, %r10, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM70-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM70-NEXT: mov.b32 %r13, %r4;
+; SM70-NEXT: @%p1 bra $L__BB4_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r12, %r4, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r12;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @xchg_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: xchg_acq_rel_v2i16_global_cta(
; SM70: {
@@ -128,6 +202,24 @@ define <8 x i16> @xchg_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
ret <8 x i16> %retval
}
+define <1 x i32> @xchg_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: xchg_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @xchg_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: xchg_acq_rel_v2i32_global_cta(
; SM70: {
@@ -195,6 +287,23 @@ define <8 x i32> @xchg_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i64> @xchg_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: xchg_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @xchg_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: xchg_acq_rel_v2i64_global_cta(
; SM70: {
@@ -265,6 +374,45 @@ define <8 x i64> @xchg_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
ret <8 x i64> %retval
}
+define <1 x i8> @add_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: add_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r6, [add_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB16_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB16_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: add_acq_rel_v2i8_global_cta(
; SM70: {
@@ -287,7 +435,7 @@ define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM70-NEXT: $L__BB12_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB17_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r15, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -304,7 +452,7 @@ define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
; SM70-NEXT: mov.b32 %r15, %r3;
-; SM70-NEXT: @%p1 bra $L__BB12_1;
+; SM70-NEXT: @%p1 bra $L__BB17_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r14, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -335,7 +483,7 @@ define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: $L__BB13_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB18_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -359,7 +507,7 @@ define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM70-NEXT: mov.b32 %r18, %r1;
-; SM70-NEXT: @%p1 bra $L__BB13_1;
+; SM70-NEXT: @%p1 bra $L__BB18_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -393,7 +541,7 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM70-NEXT: $L__BB14_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB19_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -451,7 +599,7 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB14_1;
+; SM70-NEXT: @%p1 bra $L__BB19_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -460,6 +608,45 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @add_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: add_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r6, [add_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 65535;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB20_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB20_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: add_acq_rel_v2i16_global_cta(
; SM70: {
@@ -474,7 +661,7 @@ define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NEXT: $L__BB15_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB21_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT: add.s16 %rs5, %rs4, %rs2;
@@ -483,7 +670,7 @@ define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB15_1;
+; SM70-NEXT: @%p1 bra $L__BB21_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -509,7 +696,7 @@ define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: $L__BB16_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB22_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
; SM70-NEXT: add.s16 %rs5, %rs4, %rs2;
@@ -531,7 +718,7 @@ define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB16_1;
+; SM70-NEXT: @%p1 bra $L__BB22_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -557,7 +744,7 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: $L__BB17_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB23_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
; SM70-NEXT: add.s16 %rs5, %rs4, %rs2;
@@ -579,14 +766,14 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB17_1;
+; SM70-NEXT: @%p1 bra $L__BB23_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM70-NEXT: $L__BB17_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB23_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
; SM70-NEXT: add.s16 %rs17, %rs16, %rs14;
@@ -608,7 +795,7 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB17_3;
+; SM70-NEXT: @%p2 bra $L__BB23_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -617,6 +804,24 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @add_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: add_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [add_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @add_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: add_acq_rel_v2i32_global_cta(
; SM70: {
@@ -684,6 +889,23 @@ define <8 x i32> @add_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @add_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: add_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @add_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: add_acq_rel_v2i64_global_cta(
; SM70: {
@@ -754,6 +976,45 @@ define <8 x i64> @add_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @sub_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: sub_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r6, [sub_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB32_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB32_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: sub_acq_rel_v2i8_global_cta(
; SM70: {
@@ -776,7 +1037,7 @@ define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM70-NEXT: $L__BB24_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB33_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r15, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -793,7 +1054,7 @@ define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
; SM70-NEXT: mov.b32 %r15, %r3;
-; SM70-NEXT: @%p1 bra $L__BB24_1;
+; SM70-NEXT: @%p1 bra $L__BB33_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r14, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -824,7 +1085,7 @@ define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: $L__BB25_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB34_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -848,7 +1109,7 @@ define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM70-NEXT: mov.b32 %r18, %r1;
-; SM70-NEXT: @%p1 bra $L__BB25_1;
+; SM70-NEXT: @%p1 bra $L__BB34_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -882,7 +1143,7 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM70-NEXT: $L__BB26_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB35_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -940,7 +1201,7 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB26_1;
+; SM70-NEXT: @%p1 bra $L__BB35_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -949,6 +1210,45 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @sub_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: sub_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r6, [sub_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 65535;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB36_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB36_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: sub_acq_rel_v2i16_global_cta(
; SM70: {
@@ -963,7 +1263,7 @@ define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NEXT: $L__BB27_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB37_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -972,7 +1272,7 @@ define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB27_1;
+; SM70-NEXT: @%p1 bra $L__BB37_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -998,7 +1298,7 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: $L__BB28_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB38_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -1020,7 +1320,7 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB28_1;
+; SM70-NEXT: @%p1 bra $L__BB38_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -1046,7 +1346,7 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: $L__BB29_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB39_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -1068,14 +1368,14 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB29_1;
+; SM70-NEXT: @%p1 bra $L__BB39_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM70-NEXT: $L__BB29_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB39_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
; SM70-NEXT: sub.s16 %rs17, %rs16, %rs14;
@@ -1097,7 +1397,7 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB29_3;
+; SM70-NEXT: @%p2 bra $L__BB39_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -1106,6 +1406,25 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @sub_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: sub_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<4>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [sub_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: neg.s32 %r2, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: sub_acq_rel_v2i32_global_cta(
; SM70: {
@@ -1187,6 +1506,24 @@ define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @sub_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: sub_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<5>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: neg.s64 %rd3, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd4;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: sub_acq_rel_v2i64_global_cta(
; SM70: {
@@ -1271,6 +1608,34 @@ define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @and_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: and_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<12>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r1, [and_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM70-NEXT: and.b32 %r3, %r2, 3;
+; SM70-NEXT: shl.b32 %r4, %r3, 3;
+; SM70-NEXT: mov.b32 %r5, 255;
+; SM70-NEXT: shl.b32 %r6, %r5, %r4;
+; SM70-NEXT: not.b32 %r7, %r6;
+; SM70-NEXT: shl.b32 %r8, %r1, %r4;
+; SM70-NEXT: or.b32 %r9, %r8, %r7;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM70-NEXT: shr.u32 %r11, %r10, %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r11;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @and_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: and_acq_rel_v2i8_global_cta(
; SM70: {
@@ -1311,13 +1676,13 @@ define <4 x i8> @and_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NEXT: $L__BB37_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB50_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r3, %r4, %r2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB37_1;
+; SM70-NEXT: @%p1 bra $L__BB50_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -1340,7 +1705,7 @@ define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB38_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB51_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r3, %r5, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -1356,7 +1721,7 @@ define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB38_1;
+; SM70-NEXT: @%p1 bra $L__BB51_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -1365,6 +1730,34 @@ define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @and_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: and_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<12>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r1, [and_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM70-NEXT: and.b32 %r3, %r2, 3;
+; SM70-NEXT: shl.b32 %r4, %r3, 3;
+; SM70-NEXT: mov.b32 %r5, 65535;
+; SM70-NEXT: shl.b32 %r6, %r5, %r4;
+; SM70-NEXT: not.b32 %r7, %r6;
+; SM70-NEXT: shl.b32 %r8, %r1, %r4;
+; SM70-NEXT: or.b32 %r9, %r8, %r7;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM70-NEXT: shr.u32 %r11, %r10, %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r11;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: and_acq_rel_v2i16_global_cta(
; SM70: {
@@ -1377,13 +1770,13 @@ define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NEXT: $L__BB39_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB53_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r3, %r4, %r2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB39_1;
+; SM70-NEXT: @%p1 bra $L__BB53_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -1406,7 +1799,7 @@ define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB40_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB54_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r3, %r5, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -1422,7 +1815,7 @@ define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB40_1;
+; SM70-NEXT: @%p1 bra $L__BB54_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -1445,7 +1838,7 @@ define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB41_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB55_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r5, %r9, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -1461,12 +1854,12 @@ define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB41_1;
+; SM70-NEXT: @%p1 bra $L__BB55_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM70-NEXT: $L__BB41_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB55_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r7, %r11, %r3;
; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -1482,7 +1875,7 @@ define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB41_3;
+; SM70-NEXT: @%p2 bra $L__BB55_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -1491,20 +1884,38 @@ define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
-define <2 x i32> @and_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM70-LABEL: and_acq_rel_v2i32_global_cta(
+define <1 x i32> @and_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: and_acq_rel_v1i32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b32 %r<3>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v2i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r3, [%rd1], %r1;
-; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: ld.param.b32 %r1, [and_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r2, [%rd1], %r1;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @and_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: and_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -1558,6 +1969,23 @@ define <8 x i32> @and_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @and_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: and_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [and_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @and_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: and_acq_rel_v2i64_global_cta(
; SM70: {
@@ -1628,6 +2056,46 @@ define <8 x i64> @and_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @nand_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: nand_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r6, [nand_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
+; SM70-NEXT: @%p1 bra $L__BB64_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: nand_acq_rel_v2i8_global_cta(
; SM70: {
@@ -1649,7 +2117,7 @@ define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: shl.b32 %r8, %r7, %r1;
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM70-NEXT: $L__BB48_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB65_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r18, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -1667,7 +2135,7 @@ define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM70-NEXT: mov.b32 %r18, %r3;
-; SM70-NEXT: @%p1 bra $L__BB48_1;
+; SM70-NEXT: @%p1 bra $L__BB65_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r17, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -1689,14 +2157,14 @@ define <4 x i8> @nand_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB49_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB66_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r3, %r5, %r2;
; SM70-NEXT: xor.b32 %r4, %r3, -1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM70-NEXT: mov.b32 %r5, %r1;
-; SM70-NEXT: @%p1 bra $L__BB49_1;
+; SM70-NEXT: @%p1 bra $L__BB66_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -1719,7 +2187,7 @@ define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: $L__BB50_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB67_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -1737,7 +2205,7 @@ define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB50_1;
+; SM70-NEXT: @%p1 bra $L__BB67_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -1746,6 +2214,46 @@ define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
ret <8 x i8> %retval
}
+define <1 x i16> @nand_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: nand_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r6, [nand_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 65535;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: $L__BB68_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
+; SM70-NEXT: @%p1 bra $L__BB68_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: nand_acq_rel_v2i16_global_cta(
; SM70: {
@@ -1758,14 +2266,14 @@ define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB51_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB69_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r3, %r5, %r2;
; SM70-NEXT: xor.b32 %r4, %r3, -1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM70-NEXT: mov.b32 %r5, %r1;
-; SM70-NEXT: @%p1 bra $L__BB51_1;
+; SM70-NEXT: @%p1 bra $L__BB69_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -1788,7 +2296,7 @@ define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: $L__BB52_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB70_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -1806,7 +2314,7 @@ define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB52_1;
+; SM70-NEXT: @%p1 bra $L__BB70_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -1829,7 +2337,7 @@ define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
-; SM70-NEXT: $L__BB53_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB71_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
; SM70-NEXT: cvt.u64.u32 %rd4, %r14;
@@ -1847,12 +2355,12 @@ define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB53_1;
+; SM70-NEXT: @%p1 bra $L__BB71_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r15, %rd12;
-; SM70-NEXT: $L__BB53_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB71_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd13, %r15;
; SM70-NEXT: cvt.u64.u32 %rd14, %r16;
@@ -1870,7 +2378,7 @@ define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r15, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB53_3;
+; SM70-NEXT: @%p2 bra $L__BB71_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
@@ -1879,6 +2387,34 @@ define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
ret <8 x i16> %retval
}
+define <1 x i32> @nand_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: nand_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [nand_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: not.b32 %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p1 bra $L__BB72_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: nand_acq_rel_v2i32_global_cta(
; SM70: {
@@ -1893,7 +2429,7 @@ define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB54_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB73_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -1910,7 +2446,7 @@ define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB54_1;
+; SM70-NEXT: @%p1 bra $L__BB73_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -1933,7 +2469,7 @@ define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB55_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB74_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -1950,12 +2486,12 @@ define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB55_1;
+; SM70-NEXT: @%p1 bra $L__BB74_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM70-NEXT: $L__BB55_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB74_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
@@ -1972,7 +2508,7 @@ define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB55_3;
+; SM70-NEXT: @%p2 bra $L__BB74_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -1996,7 +2532,7 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM70-NEXT: $L__BB56_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB75_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -2013,12 +2549,12 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB56_1;
+; SM70-NEXT: @%p1 bra $L__BB75_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM70-NEXT: $L__BB56_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB75_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -2035,12 +2571,12 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB56_3;
+; SM70-NEXT: @%p2 bra $L__BB75_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM70-NEXT: $L__BB56_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB75_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -2057,12 +2593,12 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM70-NEXT: @%p3 bra $L__BB56_5;
+; SM70-NEXT: @%p3 bra $L__BB75_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM70-NEXT: $L__BB56_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB75_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -2079,7 +2615,7 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM70-NEXT: @%p4 bra $L__BB56_7;
+; SM70-NEXT: @%p4 bra $L__BB75_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -2089,6 +2625,33 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i64> @nand_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: nand_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b64 %rd<7>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM70-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd4, %rd6, %rd1;
+; SM70-NEXT: not.b64 %rd5, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
+; SM70-NEXT: mov.b64 %rd6, %rd2;
+; SM70-NEXT: @%p1 bra $L__BB76_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: nand_acq_rel_v2i64_global_cta(
; SM70: {
@@ -2100,24 +2663,24 @@ define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
-; SM70-NEXT: $L__BB57_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB77_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b64 %rd6, %rd10, %rd1;
; SM70-NEXT: not.b64 %rd7, %rd6;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
; SM70-NEXT: mov.b64 %rd10, %rd3;
-; SM70-NEXT: @%p1 bra $L__BB57_1;
+; SM70-NEXT: @%p1 bra $L__BB77_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
-; SM70-NEXT: $L__BB57_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB77_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b64 %rd8, %rd11, %rd2;
; SM70-NEXT: not.b64 %rd9, %rd8;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
; SM70-NEXT: mov.b64 %rd11, %rd4;
-; SM70-NEXT: @%p2 bra $L__BB57_3;
+; SM70-NEXT: @%p2 bra $L__BB77_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -2138,44 +2701,44 @@ define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
-; SM70-NEXT: $L__BB58_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB78_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd5, %rd18;
; SM70-NEXT: and.b64 %rd10, %rd5, %rd3;
; SM70-NEXT: not.b64 %rd11, %rd10;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
; SM70-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
-; SM70-NEXT: @%p1 bra $L__BB58_1;
+; SM70-NEXT: @%p1 bra $L__BB78_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
-; SM70-NEXT: $L__BB58_3: // %atomicrmw.start5
+; SM70-NEXT: $L__BB78_3: // %atomicrmw.start5
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd6, %rd19;
; SM70-NEXT: and.b64 %rd12, %rd6, %rd4;
; SM70-NEXT: not.b64 %rd13, %rd12;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
; SM70-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
-; SM70-NEXT: @%p2 bra $L__BB58_3;
+; SM70-NEXT: @%p2 bra $L__BB78_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end4
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
-; SM70-NEXT: $L__BB58_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB78_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b64 %rd14, %rd20, %rd1;
; SM70-NEXT: not.b64 %rd15, %rd14;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
; SM70-NEXT: mov.b64 %rd20, %rd7;
-; SM70-NEXT: @%p3 bra $L__BB58_5;
+; SM70-NEXT: @%p3 bra $L__BB78_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
-; SM70-NEXT: $L__BB58_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB78_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b64 %rd16, %rd21, %rd2;
; SM70-NEXT: not.b64 %rd17, %rd16;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
; SM70-NEXT: mov.b64 %rd21, %rd8;
-; SM70-NEXT: @%p4 bra $L__BB58_7;
+; SM70-NEXT: @%p4 bra $L__BB78_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -2199,84 +2762,84 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [nand_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
-; SM70-NEXT: $L__BB59_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB79_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd7, %rd34;
; SM70-NEXT: and.b64 %rd18, %rd7, %rd5;
; SM70-NEXT: not.b64 %rd19, %rd18;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
; SM70-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
-; SM70-NEXT: @%p1 bra $L__BB59_1;
+; SM70-NEXT: @%p1 bra $L__BB79_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
-; SM70-NEXT: $L__BB59_3: // %atomicrmw.start9
+; SM70-NEXT: $L__BB79_3: // %atomicrmw.start9
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd8, %rd35;
; SM70-NEXT: and.b64 %rd20, %rd8, %rd6;
; SM70-NEXT: not.b64 %rd21, %rd20;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
; SM70-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
-; SM70-NEXT: @%p2 bra $L__BB59_3;
+; SM70-NEXT: @%p2 bra $L__BB79_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end8
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
-; SM70-NEXT: $L__BB59_5: // %atomicrmw.start20
+; SM70-NEXT: $L__BB79_5: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd9, %rd36;
; SM70-NEXT: and.b64 %rd22, %rd9, %rd3;
; SM70-NEXT: not.b64 %rd23, %rd22;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
; SM70-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
-; SM70-NEXT: @%p3 bra $L__BB59_5;
+; SM70-NEXT: @%p3 bra $L__BB79_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end19
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
-; SM70-NEXT: $L__BB59_7: // %atomicrmw.start26
+; SM70-NEXT: $L__BB79_7: // %atomicrmw.start26
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd10, %rd37;
; SM70-NEXT: and.b64 %rd24, %rd10, %rd4;
; SM70-NEXT: not.b64 %rd25, %rd24;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
; SM70-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
-; SM70-NEXT: @%p4 bra $L__BB59_7;
+; SM70-NEXT: @%p4 bra $L__BB79_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end25
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
-; SM70-NEXT: $L__BB59_9: // %atomicrmw.start41
+; SM70-NEXT: $L__BB79_9: // %atomicrmw.start41
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd13, %rd38;
; SM70-NEXT: and.b64 %rd26, %rd13, %rd1;
; SM70-NEXT: not.b64 %rd27, %rd26;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
; SM70-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
-; SM70-NEXT: @%p5 bra $L__BB59_9;
+; SM70-NEXT: @%p5 bra $L__BB79_9;
; SM70-NEXT: // %bb.10: // %atomicrmw.end40
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
-; SM70-NEXT: $L__BB59_11: // %atomicrmw.start47
+; SM70-NEXT: $L__BB79_11: // %atomicrmw.start47
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd14, %rd39;
; SM70-NEXT: and.b64 %rd28, %rd14, %rd2;
; SM70-NEXT: not.b64 %rd29, %rd28;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
; SM70-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
-; SM70-NEXT: @%p6 bra $L__BB59_11;
+; SM70-NEXT: @%p6 bra $L__BB79_11;
; SM70-NEXT: // %bb.12: // %atomicrmw.end46
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
-; SM70-NEXT: $L__BB59_13: // %atomicrmw.start58
+; SM70-NEXT: $L__BB79_13: // %atomicrmw.start58
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b64 %rd30, %rd40, %rd11;
; SM70-NEXT: not.b64 %rd31, %rd30;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
; SM70-NEXT: mov.b64 %rd40, %rd15;
-; SM70-NEXT: @%p7 bra $L__BB59_13;
+; SM70-NEXT: @%p7 bra $L__BB79_13;
; SM70-NEXT: // %bb.14: // %atomicrmw.end57
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
-; SM70-NEXT: $L__BB59_15: // %atomicrmw.start64
+; SM70-NEXT: $L__BB79_15: // %atomicrmw.start64
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b64 %rd32, %rd41, %rd12;
; SM70-NEXT: not.b64 %rd33, %rd32;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
; SM70-NEXT: mov.b64 %rd41, %rd16;
-; SM70-NEXT: @%p8 bra $L__BB59_15;
+; SM70-NEXT: @%p8 bra $L__BB79_15;
; SM70-NEXT: // %bb.16: // %atomicrmw.end63
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -2288,6 +2851,30 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
ret <8 x i64> %retval
}
+define <1 x i8> @or_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: or_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r1, [or_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM70-NEXT: and.b32 %r3, %r2, 3;
+; SM70-NEXT: shl.b32 %r4, %r3, 3;
+; SM70-NEXT: shl.b32 %r5, %r1, %r4;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM70-NEXT: shr.u32 %r7, %r6, %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r7;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @or_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: or_acq_rel_v2i8_global_cta(
; SM70: {
@@ -2324,13 +2911,13 @@ define <4 x i8> @or_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %va
; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB82_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: or.b32 %r3, %r4, %r2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB61_1;
+; SM70-NEXT: @%p1 bra $L__BB82_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -2353,7 +2940,7 @@ define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %va
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB83_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: or.b32 %r3, %r5, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -2369,7 +2956,7 @@ define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %va
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB62_1;
+; SM70-NEXT: @%p1 bra $L__BB83_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -2378,6 +2965,30 @@ define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %va
ret <8 x i8> %retval
}
+define <1 x i16> @or_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: or_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r1, [or_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM70-NEXT: and.b32 %r3, %r2, 3;
+; SM70-NEXT: shl.b32 %r4, %r3, 3;
+; SM70-NEXT: shl.b32 %r5, %r1, %r4;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM70-NEXT: shr.u32 %r7, %r6, %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r7;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: or_acq_rel_v2i16_global_cta(
; SM70: {
@@ -2390,13 +3001,13 @@ define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB85_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: or.b32 %r3, %r4, %r2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB63_1;
+; SM70-NEXT: @%p1 bra $L__BB85_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -2419,7 +3030,7 @@ define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB86_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: or.b32 %r3, %r5, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -2435,7 +3046,7 @@ define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB64_1;
+; SM70-NEXT: @%p1 bra $L__BB86_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -2458,7 +3069,7 @@ define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB87_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: or.b32 %r5, %r9, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -2474,12 +3085,12 @@ define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB65_1;
+; SM70-NEXT: @%p1 bra $L__BB87_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM70-NEXT: $L__BB65_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB87_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: or.b32 %r7, %r11, %r3;
; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -2495,7 +3106,7 @@ define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB65_3;
+; SM70-NEXT: @%p2 bra $L__BB87_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -2504,6 +3115,24 @@ define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @or_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: or_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [or_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @or_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: or_acq_rel_v2i32_global_cta(
; SM70: {
@@ -2571,6 +3200,23 @@ define <8 x i32> @or_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @or_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: or_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [or_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @or_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: or_acq_rel_v2i64_global_cta(
; SM70: {
@@ -2641,6 +3287,30 @@ define <8 x i64> @or_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @xor_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: xor_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r1, [xor_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM70-NEXT: and.b32 %r3, %r2, 3;
+; SM70-NEXT: shl.b32 %r4, %r3, 3;
+; SM70-NEXT: shl.b32 %r5, %r1, %r4;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM70-NEXT: shr.u32 %r7, %r6, %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r7;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @xor_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: xor_acq_rel_v2i8_global_cta(
; SM70: {
@@ -2677,13 +3347,13 @@ define <4 x i8> @xor_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB98_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: xor.b32 %r3, %r4, %r2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB73_1;
+; SM70-NEXT: @%p1 bra $L__BB98_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -2706,7 +3376,7 @@ define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB99_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: xor.b32 %r3, %r5, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -2722,7 +3392,7 @@ define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB74_1;
+; SM70-NEXT: @%p1 bra $L__BB99_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -2731,6 +3401,30 @@ define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @xor_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: xor_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r1, [xor_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM70-NEXT: and.b32 %r3, %r2, 3;
+; SM70-NEXT: shl.b32 %r4, %r3, 3;
+; SM70-NEXT: shl.b32 %r5, %r1, %r4;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM70-NEXT: shr.u32 %r7, %r6, %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r7;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: xor_acq_rel_v2i16_global_cta(
; SM70: {
@@ -2743,13 +3437,13 @@ define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB101_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: xor.b32 %r3, %r4, %r2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB75_1;
+; SM70-NEXT: @%p1 bra $L__BB101_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -2772,7 +3466,7 @@ define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB102_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: xor.b32 %r3, %r5, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -2788,7 +3482,7 @@ define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB76_1;
+; SM70-NEXT: @%p1 bra $L__BB102_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -2811,7 +3505,7 @@ define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB103_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: xor.b32 %r5, %r9, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -2827,12 +3521,12 @@ define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB77_1;
+; SM70-NEXT: @%p1 bra $L__BB103_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM70-NEXT: $L__BB77_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB103_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: xor.b32 %r7, %r11, %r3;
; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -2848,7 +3542,7 @@ define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB77_3;
+; SM70-NEXT: @%p2 bra $L__BB103_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -2857,6 +3551,24 @@ define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @xor_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: xor_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [xor_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @xor_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: xor_acq_rel_v2i32_global_cta(
; SM70: {
@@ -2924,6 +3636,23 @@ define <8 x i32> @xor_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @xor_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: xor_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [xor_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @xor_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: xor_acq_rel_v2i64_global_cta(
; SM70: {
@@ -2994,34 +3723,77 @@ define <8 x i64> @xor_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
-define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM70-LABEL: max_acq_rel_v2i8_global_cta(
+define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: max_acq_rel_v1i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<14>;
-; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<16>;
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [max_acq_rel_v2i8_global_cta_param_1];
-; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM70-NEXT: ld.param.b64 %rd2, [max_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.s8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: and.b32 %r6, %r5, 3;
-; SM70-NEXT: shl.b32 %r1, %r6, 3;
-; SM70-NEXT: mov.b32 %r7, 65535;
-; SM70-NEXT: shl.b32 %r8, %r7, %r1;
-; SM70-NEXT: not.b32 %r2, %r8;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM70-NEXT: cvt.s16.s8 %rs7, %rs5;
-; SM70-NEXT: cvt.s16.s8 %rs8, %rs6;
-; SM70-NEXT: $L__BB84_1: // %atomicrmw.start
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB112_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r9, %r16, %r1;
-; SM70-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM70-NEXT: shr.u32 %r8, %r15, %r1;
+; SM70-NEXT: cvt.s8.s32 %rs2, %r8;
+; SM70-NEXT: max.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: and.b32 %r10, %r9, 255;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB112_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: max_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<14>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [max_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [max_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: cvt.s16.s8 %rs7, %rs5;
+; SM70-NEXT: cvt.s16.s8 %rs8, %rs6;
+; SM70-NEXT: $L__BB113_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r16, %r1;
+; SM70-NEXT: cvt.s8.s32 %rs3, %r9;
; SM70-NEXT: bfe.s32 %r10, %r9, 8, 8;
; SM70-NEXT: cvt.s8.s32 %rs4, %r10;
; SM70-NEXT: max.s16 %rs9, %rs4, %rs8;
@@ -3036,7 +3808,7 @@ define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r16;
; SM70-NEXT: mov.b32 %r16, %r3;
-; SM70-NEXT: @%p1 bra $L__BB84_1;
+; SM70-NEXT: @%p1 bra $L__BB113_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r15, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -3062,7 +3834,7 @@ define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
-; SM70-NEXT: $L__BB85_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB114_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
; SM70-NEXT: setp.gt.s32 %p1, %r4, %r3;
@@ -3090,7 +3862,7 @@ define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NEXT: setp.ne.b32 %p5, %r1, %r26;
; SM70-NEXT: mov.b32 %r26, %r1;
-; SM70-NEXT: @%p5 bra $L__BB85_1;
+; SM70-NEXT: @%p5 bra $L__BB114_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -3116,7 +3888,7 @@ define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
-; SM70-NEXT: $L__BB86_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB115_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r49;
; SM70-NEXT: cvt.u64.u32 %rd4, %r50;
@@ -3206,7 +3978,7 @@ define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB86_1;
+; SM70-NEXT: @%p9 bra $L__BB115_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
@@ -3215,6 +3987,48 @@ define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @max_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: max_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB116_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: max.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB116_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: max_acq_rel_v2i16_global_cta(
; SM70: {
@@ -3229,7 +4043,7 @@ define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NEXT: $L__BB87_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB117_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT: max.s16 %rs5, %rs4, %rs2;
@@ -3238,7 +4052,7 @@ define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB87_1;
+; SM70-NEXT: @%p1 bra $L__BB117_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -3264,7 +4078,7 @@ define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: $L__BB88_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB118_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -3286,7 +4100,7 @@ define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB88_1;
+; SM70-NEXT: @%p1 bra $L__BB118_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -3312,7 +4126,7 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: $L__BB89_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB119_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -3334,14 +4148,14 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB89_1;
+; SM70-NEXT: @%p1 bra $L__BB119_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM70-NEXT: $L__BB89_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB119_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -3363,7 +4177,7 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB89_3;
+; SM70-NEXT: @%p2 bra $L__BB119_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -3372,6 +4186,24 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @max_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: max_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [max_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @max_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: max_acq_rel_v2i32_global_cta(
; SM70: {
@@ -3439,6 +4271,23 @@ define <8 x i32> @max_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @max_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: max_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @max_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: max_acq_rel_v2i64_global_cta(
; SM70: {
@@ -3509,6 +4358,49 @@ define <8 x i64> @max_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: min_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.s8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB128_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r15, %r1;
+; SM70-NEXT: cvt.s8.s32 %rs2, %r8;
+; SM70-NEXT: min.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: and.b32 %r10, %r9, 255;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB128_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: min_acq_rel_v2i8_global_cta(
; SM70: {
@@ -3533,7 +4425,7 @@ define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
; SM70-NEXT: cvt.s16.s8 %rs7, %rs5;
; SM70-NEXT: cvt.s16.s8 %rs8, %rs6;
-; SM70-NEXT: $L__BB96_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB129_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r16, %r1;
; SM70-NEXT: cvt.s8.s32 %rs3, %r9;
@@ -3551,7 +4443,7 @@ define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r16;
; SM70-NEXT: mov.b32 %r16, %r3;
-; SM70-NEXT: @%p1 bra $L__BB96_1;
+; SM70-NEXT: @%p1 bra $L__BB129_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r15, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -3577,7 +4469,7 @@ define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
-; SM70-NEXT: $L__BB97_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB130_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
; SM70-NEXT: setp.le.s32 %p1, %r4, %r3;
@@ -3605,7 +4497,7 @@ define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NEXT: setp.ne.b32 %p5, %r1, %r26;
; SM70-NEXT: mov.b32 %r26, %r1;
-; SM70-NEXT: @%p5 bra $L__BB97_1;
+; SM70-NEXT: @%p5 bra $L__BB130_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -3631,7 +4523,7 @@ define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
-; SM70-NEXT: $L__BB98_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB131_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r49;
; SM70-NEXT: cvt.u64.u32 %rd4, %r50;
@@ -3721,7 +4613,7 @@ define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB98_1;
+; SM70-NEXT: @%p9 bra $L__BB131_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
@@ -3730,6 +4622,48 @@ define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @min_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: min_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB132_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: min.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB132_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: min_acq_rel_v2i16_global_cta(
; SM70: {
@@ -3744,7 +4678,7 @@ define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NEXT: $L__BB99_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB133_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT: min.s16 %rs5, %rs4, %rs2;
@@ -3753,7 +4687,7 @@ define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB99_1;
+; SM70-NEXT: @%p1 bra $L__BB133_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -3779,7 +4713,7 @@ define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: $L__BB100_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB134_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -3801,7 +4735,7 @@ define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB100_1;
+; SM70-NEXT: @%p1 bra $L__BB134_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -3827,7 +4761,7 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: $L__BB101_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB135_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -3849,14 +4783,14 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB101_1;
+; SM70-NEXT: @%p1 bra $L__BB135_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM70-NEXT: $L__BB101_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB135_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -3878,7 +4812,7 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB101_3;
+; SM70-NEXT: @%p2 bra $L__BB135_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -3887,6 +4821,24 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @min_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: min_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [min_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @min_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: min_acq_rel_v2i32_global_cta(
; SM70: {
@@ -3954,6 +4906,23 @@ define <8 x i32> @min_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @min_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: min_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @min_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: min_acq_rel_v2i64_global_cta(
; SM70: {
@@ -4024,6 +4993,49 @@ define <8 x i64> @min_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @umax_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: umax_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [umax_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB144_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: and.b16 %rs3, %rs2, 255;
+; SM70-NEXT: max.u16 %rs4, %rs3, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB144_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: umax_acq_rel_v2i8_global_cta(
; SM70: {
@@ -4046,7 +5058,7 @@ define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r4;
-; SM70-NEXT: $L__BB108_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB145_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r17, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -4065,7 +5077,7 @@ define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17;
; SM70-NEXT: mov.b32 %r17, %r3;
-; SM70-NEXT: @%p1 bra $L__BB108_1;
+; SM70-NEXT: @%p1 bra $L__BB145_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r16, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -4091,7 +5103,7 @@ define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
-; SM70-NEXT: $L__BB109_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB146_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
; SM70-NEXT: setp.gt.u32 %p1, %r4, %r3;
@@ -4111,7 +5123,7 @@ define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT: setp.ne.b32 %p5, %r1, %r18;
; SM70-NEXT: mov.b32 %r18, %r1;
-; SM70-NEXT: @%p5 bra $L__BB109_1;
+; SM70-NEXT: @%p5 bra $L__BB146_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -4139,7 +5151,7 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
-; SM70-NEXT: $L__BB110_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB147_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -4211,7 +5223,7 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB110_1;
+; SM70-NEXT: @%p9 bra $L__BB147_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -4220,6 +5232,48 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
ret <8 x i8> %retval
}
+define <1 x i16> @umax_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: umax_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB148_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB148_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: umax_acq_rel_v2i16_global_cta(
; SM70: {
@@ -4234,7 +5288,7 @@ define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NEXT: $L__BB111_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB149_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT: max.u16 %rs5, %rs4, %rs2;
@@ -4243,7 +5297,7 @@ define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB111_1;
+; SM70-NEXT: @%p1 bra $L__BB149_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -4269,7 +5323,7 @@ define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: $L__BB112_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB150_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -4291,7 +5345,7 @@ define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB112_1;
+; SM70-NEXT: @%p1 bra $L__BB150_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -4317,7 +5371,7 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: $L__BB113_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB151_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -4339,14 +5393,14 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB113_1;
+; SM70-NEXT: @%p1 bra $L__BB151_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM70-NEXT: $L__BB113_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB151_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -4368,7 +5422,7 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB113_3;
+; SM70-NEXT: @%p2 bra $L__BB151_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -4377,6 +5431,24 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
ret <8 x i16> %retval
}
+define <1 x i32> @umax_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: umax_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [umax_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @umax_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: umax_acq_rel_v2i32_global_cta(
; SM70: {
@@ -4444,6 +5516,23 @@ define <8 x i32> @umax_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i64> @umax_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: umax_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @umax_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: umax_acq_rel_v2i64_global_cta(
; SM70: {
@@ -4514,6 +5603,49 @@ define <8 x i64> @umax_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
ret <8 x i64> %retval
}
+define <1 x i8> @umin_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: umin_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [umin_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB160_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: and.b16 %rs3, %rs2, 255;
+; SM70-NEXT: min.u16 %rs4, %rs3, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB160_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: umin_acq_rel_v2i8_global_cta(
; SM70: {
@@ -4536,7 +5668,7 @@ define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r4;
-; SM70-NEXT: $L__BB120_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB161_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r17, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -4555,7 +5687,7 @@ define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17;
; SM70-NEXT: mov.b32 %r17, %r3;
-; SM70-NEXT: @%p1 bra $L__BB120_1;
+; SM70-NEXT: @%p1 bra $L__BB161_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r16, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -4581,7 +5713,7 @@ define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
-; SM70-NEXT: $L__BB121_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB162_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
; SM70-NEXT: setp.le.u32 %p1, %r4, %r3;
@@ -4601,7 +5733,7 @@ define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT: setp.ne.b32 %p5, %r1, %r18;
; SM70-NEXT: mov.b32 %r18, %r1;
-; SM70-NEXT: @%p5 bra $L__BB121_1;
+; SM70-NEXT: @%p5 bra $L__BB162_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -4629,7 +5761,7 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
-; SM70-NEXT: $L__BB122_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB163_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -4701,7 +5833,7 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB122_1;
+; SM70-NEXT: @%p9 bra $L__BB163_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -4710,6 +5842,48 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
ret <8 x i8> %retval
}
+define <1 x i16> @umin_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: umin_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB164_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: min.u16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB164_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: umin_acq_rel_v2i16_global_cta(
; SM70: {
@@ -4724,7 +5898,7 @@ define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NEXT: $L__BB123_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB165_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT: min.u16 %rs5, %rs4, %rs2;
@@ -4733,7 +5907,7 @@ define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB123_1;
+; SM70-NEXT: @%p1 bra $L__BB165_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -4759,7 +5933,7 @@ define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: $L__BB124_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB166_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -4781,7 +5955,7 @@ define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB124_1;
+; SM70-NEXT: @%p1 bra $L__BB166_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -4807,7 +5981,7 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: $L__BB125_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB167_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -4829,14 +6003,14 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB125_1;
+; SM70-NEXT: @%p1 bra $L__BB167_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM70-NEXT: $L__BB125_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB167_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -4858,7 +6032,7 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB125_3;
+; SM70-NEXT: @%p2 bra $L__BB167_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -4867,6 +6041,24 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
ret <8 x i16> %retval
}
+define <1 x i32> @umin_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: umin_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [umin_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @umin_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: umin_acq_rel_v2i32_global_cta(
; SM70: {
@@ -4934,7 +6126,24 @@ define <8 x i32> @umin_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
-define <2 x i64> @umin_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+define <1 x i64> @umin_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: umin_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @umin_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: umin_acq_rel_v2i64_global_cta(
; SM70: {
; SM70-NEXT: .reg .b64 %rd<6>;
@@ -5004,6 +6213,52 @@ define <8 x i64> @umin_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
ret <8 x i64> %retval
}
+define <1 x i8> @uinc_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<6>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [uinc_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB176_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: and.b16 %rs3, %rs2, 255;
+; SM70-NEXT: add.s16 %rs4, %rs2, 1;
+; SM70-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs5, 0, %rs4, %p1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM70-NEXT: and.b32 %r10, %r9, 255;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p2 bra $L__BB176_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v2i8_global_cta(
; SM70: {
@@ -5026,7 +6281,7 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT: mov.b32 {%rs8, %rs9}, %r4;
-; SM70-NEXT: $L__BB132_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB177_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r15, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -5048,7 +6303,7 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM70-NEXT: setp.ne.b32 %p3, %r3, %r15;
; SM70-NEXT: mov.b32 %r15, %r3;
-; SM70-NEXT: @%p3 bra $L__BB132_1;
+; SM70-NEXT: @%p3 bra $L__BB177_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r14, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -5075,7 +6330,7 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
-; SM70-NEXT: $L__BB133_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB178_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r3, %r22, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
@@ -5107,7 +6362,7 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
; SM70-NEXT: setp.ne.b32 %p5, %r1, %r22;
; SM70-NEXT: mov.b32 %r22, %r1;
-; SM70-NEXT: @%p5 bra $L__BB133_1;
+; SM70-NEXT: @%p5 bra $L__BB178_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -5133,7 +6388,7 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
-; SM70-NEXT: $L__BB134_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB179_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -5207,7 +6462,7 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB134_1;
+; SM70-NEXT: @%p9 bra $L__BB179_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -5216,6 +6471,50 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i8> %retval
}
+define <1 x i16> @uinc_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB180_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: add.s16 %rs3, %rs2, 1;
+; SM70-NEXT: setp.ge.u16 %p1, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs4, 0, %rs3, %p1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p2 bra $L__BB180_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v2i16_global_cta(
; SM70: {
@@ -5230,7 +6529,7 @@ define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NEXT: $L__BB135_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB181_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; SM70-NEXT: add.s16 %rs3, %rs1, 1;
@@ -5243,7 +6542,7 @@ define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p3, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p3 bra $L__BB135_1;
+; SM70-NEXT: @%p3 bra $L__BB181_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -5269,7 +6568,7 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r1;
-; SM70-NEXT: $L__BB136_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB182_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -5299,7 +6598,7 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB136_1;
+; SM70-NEXT: @%p5 bra $L__BB182_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -5325,7 +6624,7 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r1;
-; SM70-NEXT: $L__BB137_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB183_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -5355,12 +6654,12 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB137_1;
+; SM70-NEXT: @%p5 bra $L__BB183_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM70-NEXT: $L__BB137_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB183_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -5392,7 +6691,7 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd16;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p10 bra $L__BB137_3;
+; SM70-NEXT: @%p10 bra $L__BB183_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -5401,6 +6700,24 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i16> %retval
}
+define <1 x i32> @uinc_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [uinc_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @uinc_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v2i32_global_cta(
; SM70: {
@@ -5468,6 +6785,34 @@ define <8 x i32> @uinc_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i32> %retval
}
+define <1 x i64> @uinc_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<7>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM70-NEXT: $L__BB188_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd4, %rd6, 1;
+; SM70-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM70-NEXT: selp.b64 %rd5, 0, %rd4, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM70-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
+; SM70-NEXT: mov.b64 %rd6, %rd2;
+; SM70-NEXT: @%p2 bra $L__BB188_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v2i64_global_cta(
; SM70: {
@@ -5479,7 +6824,7 @@ define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
-; SM70-NEXT: $L__BB141_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB189_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd6, %rd10, 1;
; SM70-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
@@ -5487,10 +6832,10 @@ define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
; SM70-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
; SM70-NEXT: mov.b64 %rd10, %rd3;
-; SM70-NEXT: @%p2 bra $L__BB141_1;
+; SM70-NEXT: @%p2 bra $L__BB189_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
-; SM70-NEXT: $L__BB141_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB189_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd8, %rd11, 1;
; SM70-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
@@ -5498,7 +6843,7 @@ define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM70-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
; SM70-NEXT: mov.b64 %rd11, %rd4;
-; SM70-NEXT: @%p4 bra $L__BB141_3;
+; SM70-NEXT: @%p4 bra $L__BB189_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -5519,7 +6864,7 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
-; SM70-NEXT: $L__BB142_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB190_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd5, %rd18;
; SM70-NEXT: add.s64 %rd10, %rd5, 1;
@@ -5527,10 +6872,10 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: selp.b64 %rd11, 0, %rd10, %p1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
; SM70-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
-; SM70-NEXT: @%p2 bra $L__BB142_1;
+; SM70-NEXT: @%p2 bra $L__BB190_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
-; SM70-NEXT: $L__BB142_3: // %atomicrmw.start5
+; SM70-NEXT: $L__BB190_3: // %atomicrmw.start5
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd6, %rd19;
; SM70-NEXT: add.s64 %rd12, %rd6, 1;
@@ -5538,10 +6883,10 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: selp.b64 %rd13, 0, %rd12, %p3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
; SM70-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
-; SM70-NEXT: @%p4 bra $L__BB142_3;
+; SM70-NEXT: @%p4 bra $L__BB190_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end4
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
-; SM70-NEXT: $L__BB142_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB190_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd14, %rd20, 1;
; SM70-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
@@ -5549,10 +6894,10 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
; SM70-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
; SM70-NEXT: mov.b64 %rd20, %rd7;
-; SM70-NEXT: @%p6 bra $L__BB142_5;
+; SM70-NEXT: @%p6 bra $L__BB190_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
-; SM70-NEXT: $L__BB142_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB190_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd16, %rd21, 1;
; SM70-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
@@ -5560,7 +6905,7 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM70-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
; SM70-NEXT: mov.b64 %rd21, %rd8;
-; SM70-NEXT: @%p8 bra $L__BB142_7;
+; SM70-NEXT: @%p8 bra $L__BB190_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -5584,7 +6929,7 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
-; SM70-NEXT: $L__BB143_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB191_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd7, %rd34;
; SM70-NEXT: add.s64 %rd18, %rd7, 1;
@@ -5592,10 +6937,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd19, 0, %rd18, %p1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
; SM70-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
-; SM70-NEXT: @%p2 bra $L__BB143_1;
+; SM70-NEXT: @%p2 bra $L__BB191_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
-; SM70-NEXT: $L__BB143_3: // %atomicrmw.start9
+; SM70-NEXT: $L__BB191_3: // %atomicrmw.start9
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd8, %rd35;
; SM70-NEXT: add.s64 %rd20, %rd8, 1;
@@ -5603,10 +6948,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd21, 0, %rd20, %p3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
; SM70-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
-; SM70-NEXT: @%p4 bra $L__BB143_3;
+; SM70-NEXT: @%p4 bra $L__BB191_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end8
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
-; SM70-NEXT: $L__BB143_5: // %atomicrmw.start20
+; SM70-NEXT: $L__BB191_5: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd9, %rd36;
; SM70-NEXT: add.s64 %rd22, %rd9, 1;
@@ -5614,10 +6959,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd23, 0, %rd22, %p5;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
; SM70-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
-; SM70-NEXT: @%p6 bra $L__BB143_5;
+; SM70-NEXT: @%p6 bra $L__BB191_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end19
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
-; SM70-NEXT: $L__BB143_7: // %atomicrmw.start26
+; SM70-NEXT: $L__BB191_7: // %atomicrmw.start26
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd10, %rd37;
; SM70-NEXT: add.s64 %rd24, %rd10, 1;
@@ -5625,10 +6970,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd25, 0, %rd24, %p7;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
; SM70-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
-; SM70-NEXT: @%p8 bra $L__BB143_7;
+; SM70-NEXT: @%p8 bra $L__BB191_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end25
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
-; SM70-NEXT: $L__BB143_9: // %atomicrmw.start41
+; SM70-NEXT: $L__BB191_9: // %atomicrmw.start41
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd13, %rd38;
; SM70-NEXT: add.s64 %rd26, %rd13, 1;
@@ -5636,10 +6981,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd27, 0, %rd26, %p9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
; SM70-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
-; SM70-NEXT: @%p10 bra $L__BB143_9;
+; SM70-NEXT: @%p10 bra $L__BB191_9;
; SM70-NEXT: // %bb.10: // %atomicrmw.end40
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
-; SM70-NEXT: $L__BB143_11: // %atomicrmw.start47
+; SM70-NEXT: $L__BB191_11: // %atomicrmw.start47
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd14, %rd39;
; SM70-NEXT: add.s64 %rd28, %rd14, 1;
@@ -5647,10 +6992,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd29, 0, %rd28, %p11;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
; SM70-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
-; SM70-NEXT: @%p12 bra $L__BB143_11;
+; SM70-NEXT: @%p12 bra $L__BB191_11;
; SM70-NEXT: // %bb.12: // %atomicrmw.end46
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
-; SM70-NEXT: $L__BB143_13: // %atomicrmw.start58
+; SM70-NEXT: $L__BB191_13: // %atomicrmw.start58
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd30, %rd40, 1;
; SM70-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
@@ -5658,10 +7003,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
; SM70-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
; SM70-NEXT: mov.b64 %rd40, %rd15;
-; SM70-NEXT: @%p14 bra $L__BB143_13;
+; SM70-NEXT: @%p14 bra $L__BB191_13;
; SM70-NEXT: // %bb.14: // %atomicrmw.end57
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
-; SM70-NEXT: $L__BB143_15: // %atomicrmw.start64
+; SM70-NEXT: $L__BB191_15: // %atomicrmw.start64
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd32, %rd41, 1;
; SM70-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
@@ -5669,7 +7014,7 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM70-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
; SM70-NEXT: mov.b64 %rd41, %rd16;
-; SM70-NEXT: @%p16 bra $L__BB143_15;
+; SM70-NEXT: @%p16 bra $L__BB191_15;
; SM70-NEXT: // %bb.16: // %atomicrmw.end63
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -5681,6 +7026,54 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i64> %retval
}
+define <1 x i8> @udec_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [udec_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB192_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: and.b16 %rs3, %rs2, 255;
+; SM70-NEXT: add.s16 %rs4, %rs2, -1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs3, 0;
+; SM70-NEXT: setp.gt.u16 %p2, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs5, %rs1, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-NEXT: and.b32 %r10, %r9, 255;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p3 bra $L__BB192_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v2i8_global_cta(
; SM70: {
@@ -5703,7 +7096,7 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT: mov.b32 {%rs8, %rs9}, %r4;
-; SM70-NEXT: $L__BB144_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB193_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r15, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -5729,7 +7122,7 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM70-NEXT: setp.ne.b32 %p5, %r3, %r15;
; SM70-NEXT: mov.b32 %r15, %r3;
-; SM70-NEXT: @%p5 bra $L__BB144_1;
+; SM70-NEXT: @%p5 bra $L__BB193_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r14, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -5756,7 +7149,7 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
-; SM70-NEXT: $L__BB145_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB194_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r3, %r26, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
@@ -5796,7 +7189,7 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NEXT: setp.ne.b32 %p9, %r1, %r26;
; SM70-NEXT: mov.b32 %r26, %r1;
-; SM70-NEXT: @%p9 bra $L__BB145_1;
+; SM70-NEXT: @%p9 bra $L__BB194_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -5824,7 +7217,7 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs17, %r11;
; SM70-NEXT: cvt.u16.u32 %rs18, %r12;
-; SM70-NEXT: $L__BB146_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB195_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -5920,7 +7313,7 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p17, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p17 bra $L__BB146_1;
+; SM70-NEXT: @%p17 bra $L__BB195_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -5929,6 +7322,52 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i8> %retval
}
+define <1 x i16> @udec_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<6>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB196_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: add.s16 %rs3, %rs2, -1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM70-NEXT: setp.gt.u16 %p2, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs4, %rs1, %rs3, %p2;
+; SM70-NEXT: selp.b16 %rs5, %rs1, %rs4, %p1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p3, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p3 bra $L__BB196_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v2i16_global_cta(
; SM70: {
@@ -5943,7 +7382,7 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NEXT: $L__BB147_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB197_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; SM70-NEXT: add.s16 %rs3, %rs1, -1;
@@ -5960,7 +7399,7 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p5, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p5 bra $L__BB147_1;
+; SM70-NEXT: @%p5 bra $L__BB197_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -5986,7 +7425,7 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r1;
-; SM70-NEXT: $L__BB148_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB198_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -6024,7 +7463,7 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB148_1;
+; SM70-NEXT: @%p9 bra $L__BB198_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -6050,7 +7489,7 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r1;
-; SM70-NEXT: $L__BB149_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB199_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -6088,12 +7527,12 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB149_1;
+; SM70-NEXT: @%p9 bra $L__BB199_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM70-NEXT: $L__BB149_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB199_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -6133,7 +7572,7 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd16;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p18 bra $L__BB149_3;
+; SM70-NEXT: @%p18 bra $L__BB199_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -6142,6 +7581,24 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i16> %retval
}
+define <1 x i32> @udec_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [udec_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @udec_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v2i32_global_cta(
; SM70: {
@@ -6209,6 +7666,36 @@ define <8 x i32> @udec_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i32> %retval
}
+define <1 x i64> @udec_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b64 %rd<8>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd7, [%rd3];
+; SM70-NEXT: $L__BB204_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd4, %rd7, -1;
+; SM70-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM70-NEXT: setp.gt.u64 %p2, %rd7, %rd1;
+; SM70-NEXT: selp.b64 %rd5, %rd1, %rd4, %p2;
+; SM70-NEXT: selp.b64 %rd6, %rd1, %rd5, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd7, %rd6;
+; SM70-NEXT: setp.ne.b64 %p3, %rd2, %rd7;
+; SM70-NEXT: mov.b64 %rd7, %rd2;
+; SM70-NEXT: @%p3 bra $L__BB204_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v2i64_global_cta(
; SM70: {
@@ -6220,7 +7707,7 @@ define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd5];
-; SM70-NEXT: $L__BB153_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB205_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd6, %rd12, -1;
; SM70-NEXT: setp.eq.b64 %p1, %rd12, 0;
@@ -6230,10 +7717,10 @@ define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd12, %rd8;
; SM70-NEXT: setp.ne.b64 %p3, %rd3, %rd12;
; SM70-NEXT: mov.b64 %rd12, %rd3;
-; SM70-NEXT: @%p3 bra $L__BB153_1;
+; SM70-NEXT: @%p3 bra $L__BB205_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd5+8];
-; SM70-NEXT: $L__BB153_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB205_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd9, %rd13, -1;
; SM70-NEXT: setp.eq.b64 %p4, %rd13, 0;
@@ -6243,7 +7730,7 @@ define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd13, %rd11;
; SM70-NEXT: setp.ne.b64 %p6, %rd4, %rd13;
; SM70-NEXT: mov.b64 %rd13, %rd4;
-; SM70-NEXT: @%p6 bra $L__BB153_3;
+; SM70-NEXT: @%p6 bra $L__BB205_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -6264,7 +7751,7 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd9];
-; SM70-NEXT: $L__BB154_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB206_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd5, %rd22;
; SM70-NEXT: add.s64 %rd10, %rd5, -1;
@@ -6274,10 +7761,10 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: selp.b64 %rd12, %rd3, %rd11, %p1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd22, [%rd9], %rd5, %rd12;
; SM70-NEXT: setp.ne.b64 %p3, %rd22, %rd5;
-; SM70-NEXT: @%p3 bra $L__BB154_1;
+; SM70-NEXT: @%p3 bra $L__BB206_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd23, [%rd9+8];
-; SM70-NEXT: $L__BB154_3: // %atomicrmw.start5
+; SM70-NEXT: $L__BB206_3: // %atomicrmw.start5
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd6, %rd23;
; SM70-NEXT: add.s64 %rd13, %rd6, -1;
@@ -6287,10 +7774,10 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: selp.b64 %rd15, %rd4, %rd14, %p4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd9+8], %rd6, %rd15;
; SM70-NEXT: setp.ne.b64 %p6, %rd23, %rd6;
-; SM70-NEXT: @%p6 bra $L__BB154_3;
+; SM70-NEXT: @%p6 bra $L__BB206_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end4
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd9+16];
-; SM70-NEXT: $L__BB154_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB206_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd16, %rd24, -1;
; SM70-NEXT: setp.eq.b64 %p7, %rd24, 0;
@@ -6300,10 +7787,10 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd24, %rd18;
; SM70-NEXT: setp.ne.b64 %p9, %rd7, %rd24;
; SM70-NEXT: mov.b64 %rd24, %rd7;
-; SM70-NEXT: @%p9 bra $L__BB154_5;
+; SM70-NEXT: @%p9 bra $L__BB206_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd25, [%rd9+24];
-; SM70-NEXT: $L__BB154_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB206_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd19, %rd25, -1;
; SM70-NEXT: setp.eq.b64 %p10, %rd25, 0;
@@ -6313,7 +7800,7 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd25, %rd21;
; SM70-NEXT: setp.ne.b64 %p12, %rd8, %rd25;
; SM70-NEXT: mov.b64 %rd25, %rd8;
-; SM70-NEXT: @%p12 bra $L__BB154_7;
+; SM70-NEXT: @%p12 bra $L__BB206_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -6337,7 +7824,7 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd42, [%rd17];
-; SM70-NEXT: $L__BB155_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB207_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd7, %rd42;
; SM70-NEXT: add.s64 %rd18, %rd7, -1;
@@ -6347,10 +7834,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd20, %rd5, %rd19, %p1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd42, [%rd17], %rd7, %rd20;
; SM70-NEXT: setp.ne.b64 %p3, %rd42, %rd7;
-; SM70-NEXT: @%p3 bra $L__BB155_1;
+; SM70-NEXT: @%p3 bra $L__BB207_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd43, [%rd17+8];
-; SM70-NEXT: $L__BB155_3: // %atomicrmw.start9
+; SM70-NEXT: $L__BB207_3: // %atomicrmw.start9
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd8, %rd43;
; SM70-NEXT: add.s64 %rd21, %rd8, -1;
@@ -6360,10 +7847,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd23, %rd6, %rd22, %p4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd43, [%rd17+8], %rd8, %rd23;
; SM70-NEXT: setp.ne.b64 %p6, %rd43, %rd8;
-; SM70-NEXT: @%p6 bra $L__BB155_3;
+; SM70-NEXT: @%p6 bra $L__BB207_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end8
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd44, [%rd17+16];
-; SM70-NEXT: $L__BB155_5: // %atomicrmw.start20
+; SM70-NEXT: $L__BB207_5: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd9, %rd44;
; SM70-NEXT: add.s64 %rd24, %rd9, -1;
@@ -6373,10 +7860,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd26, %rd3, %rd25, %p7;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd44, [%rd17+16], %rd9, %rd26;
; SM70-NEXT: setp.ne.b64 %p9, %rd44, %rd9;
-; SM70-NEXT: @%p9 bra $L__BB155_5;
+; SM70-NEXT: @%p9 bra $L__BB207_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end19
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd45, [%rd17+24];
-; SM70-NEXT: $L__BB155_7: // %atomicrmw.start26
+; SM70-NEXT: $L__BB207_7: // %atomicrmw.start26
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd10, %rd45;
; SM70-NEXT: add.s64 %rd27, %rd10, -1;
@@ -6386,10 +7873,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd29, %rd4, %rd28, %p10;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd17+24], %rd10, %rd29;
; SM70-NEXT: setp.ne.b64 %p12, %rd45, %rd10;
-; SM70-NEXT: @%p12 bra $L__BB155_7;
+; SM70-NEXT: @%p12 bra $L__BB207_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end25
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd46, [%rd17+32];
-; SM70-NEXT: $L__BB155_9: // %atomicrmw.start41
+; SM70-NEXT: $L__BB207_9: // %atomicrmw.start41
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd13, %rd46;
; SM70-NEXT: add.s64 %rd30, %rd13, -1;
@@ -6399,10 +7886,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd32, %rd1, %rd31, %p13;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd46, [%rd17+32], %rd13, %rd32;
; SM70-NEXT: setp.ne.b64 %p15, %rd46, %rd13;
-; SM70-NEXT: @%p15 bra $L__BB155_9;
+; SM70-NEXT: @%p15 bra $L__BB207_9;
; SM70-NEXT: // %bb.10: // %atomicrmw.end40
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd47, [%rd17+40];
-; SM70-NEXT: $L__BB155_11: // %atomicrmw.start47
+; SM70-NEXT: $L__BB207_11: // %atomicrmw.start47
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd14, %rd47;
; SM70-NEXT: add.s64 %rd33, %rd14, -1;
@@ -6412,10 +7899,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd35, %rd2, %rd34, %p16;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd47, [%rd17+40], %rd14, %rd35;
; SM70-NEXT: setp.ne.b64 %p18, %rd47, %rd14;
-; SM70-NEXT: @%p18 bra $L__BB155_11;
+; SM70-NEXT: @%p18 bra $L__BB207_11;
; SM70-NEXT: // %bb.12: // %atomicrmw.end46
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd48, [%rd17+48];
-; SM70-NEXT: $L__BB155_13: // %atomicrmw.start58
+; SM70-NEXT: $L__BB207_13: // %atomicrmw.start58
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd36, %rd48, -1;
; SM70-NEXT: setp.eq.b64 %p19, %rd48, 0;
@@ -6425,10 +7912,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd48, %rd38;
; SM70-NEXT: setp.ne.b64 %p21, %rd15, %rd48;
; SM70-NEXT: mov.b64 %rd48, %rd15;
-; SM70-NEXT: @%p21 bra $L__BB155_13;
+; SM70-NEXT: @%p21 bra $L__BB207_13;
; SM70-NEXT: // %bb.14: // %atomicrmw.end57
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd49, [%rd17+56];
-; SM70-NEXT: $L__BB155_15: // %atomicrmw.start64
+; SM70-NEXT: $L__BB207_15: // %atomicrmw.start64
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: add.s64 %rd39, %rd49, -1;
; SM70-NEXT: setp.eq.b64 %p22, %rd49, 0;
@@ -6438,7 +7925,7 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd49, %rd41;
; SM70-NEXT: setp.ne.b64 %p24, %rd16, %rd49;
; SM70-NEXT: mov.b64 %rd49, %rd16;
-; SM70-NEXT: @%p24 bra $L__BB155_15;
+; SM70-NEXT: @%p24 bra $L__BB207_15;
; SM70-NEXT: // %bb.16: // %atomicrmw.end63
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -6450,6 +7937,52 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i64> %retval
}
+define <1 x i8> @usub_cond_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<6>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [usub_cond_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB208_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: and.b16 %rs3, %rs2, 255;
+; SM70-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM70-NEXT: sub.s16 %rs4, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM70-NEXT: and.b32 %r10, %r9, 255;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p2 bra $L__BB208_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: usub_cond_acq_rel_v2i8_global_cta(
; SM70: {
@@ -6472,7 +8005,7 @@ define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT: mov.b32 {%rs6, %rs7}, %r4;
-; SM70-NEXT: $L__BB156_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB209_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r15, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -6494,7 +8027,7 @@ define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM70-NEXT: setp.ne.b32 %p3, %r3, %r15;
; SM70-NEXT: mov.b32 %r15, %r3;
-; SM70-NEXT: @%p3 bra $L__BB156_1;
+; SM70-NEXT: @%p3 bra $L__BB209_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r14, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -6525,7 +8058,7 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: cvt.u16.u32 %rs2, %r7;
; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
; SM70-NEXT: cvt.u16.u32 %rs4, %r3;
-; SM70-NEXT: $L__BB157_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB210_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r22, 0, 0x7770U;
; SM70-NEXT: setp.ge.u32 %p1, %r4, %r3;
@@ -6557,7 +8090,7 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
; SM70-NEXT: setp.ne.b32 %p5, %r1, %r22;
; SM70-NEXT: mov.b32 %r22, %r1;
-; SM70-NEXT: @%p5 bra $L__BB157_1;
+; SM70-NEXT: @%p5 bra $L__BB210_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -6585,7 +8118,7 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs13, %r5;
; SM70-NEXT: cvt.u16.u32 %rs15, %r3;
-; SM70-NEXT: $L__BB158_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB211_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -6665,7 +8198,7 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB158_1;
+; SM70-NEXT: @%p9 bra $L__BB211_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -6674,6 +8207,50 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i8> %retval
}
+define <1 x i16> @usub_cond_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB212_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: setp.ge.u16 %p1, %rs2, %rs1;
+; SM70-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs4, %rs3, %rs2, %p1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p2 bra $L__BB212_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: usub_cond_acq_rel_v2i16_global_cta(
; SM70: {
@@ -6688,7 +8265,7 @@ define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NEXT: $L__BB159_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB213_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
@@ -6701,7 +8278,7 @@ define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p3, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p3 bra $L__BB159_1;
+; SM70-NEXT: @%p3 bra $L__BB213_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -6727,7 +8304,7 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; SM70-NEXT: $L__BB160_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB214_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -6757,7 +8334,7 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB160_1;
+; SM70-NEXT: @%p5 bra $L__BB214_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -6783,7 +8360,7 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; SM70-NEXT: $L__BB161_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB215_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -6813,12 +8390,12 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB161_1;
+; SM70-NEXT: @%p5 bra $L__BB215_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM70-NEXT: $L__BB161_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB215_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
@@ -6850,7 +8427,7 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd16;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p10 bra $L__BB161_3;
+; SM70-NEXT: @%p10 bra $L__BB215_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -6859,6 +8436,35 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i16> %retval
}
+define <1 x i32> @usub_cond_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [usub_cond_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB216_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u32 %p1, %r5, %r1;
+; SM70-NEXT: sub.s32 %r3, %r5, %r1;
+; SM70-NEXT: selp.b32 %r4, %r3, %r5, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p2, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p2 bra $L__BB216_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: usub_cond_acq_rel_v2i32_global_cta(
; SM70: {
@@ -6873,7 +8479,7 @@ define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: $L__BB162_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB217_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -6893,7 +8499,7 @@ define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM70-NEXT: @%p3 bra $L__BB162_1;
+; SM70-NEXT: @%p3 bra $L__BB217_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -6916,7 +8522,7 @@ define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
-; SM70-NEXT: $L__BB163_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB218_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
; SM70-NEXT: cvt.u64.u32 %rd4, %r14;
@@ -6936,12 +8542,12 @@ define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM70-NEXT: @%p3 bra $L__BB163_1;
+; SM70-NEXT: @%p3 bra $L__BB218_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r15, %rd12;
-; SM70-NEXT: $L__BB163_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB218_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd13, %r15;
; SM70-NEXT: cvt.u64.u32 %rd14, %r16;
@@ -6961,7 +8567,7 @@ define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: setp.ne.b64 %p6, %rd21, %rd16;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r15, %rd21;
-; SM70-NEXT: @%p6 bra $L__BB163_3;
+; SM70-NEXT: @%p6 bra $L__BB218_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
@@ -6985,7 +8591,7 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
-; SM70-NEXT: $L__BB164_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB219_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
; SM70-NEXT: cvt.u64.u32 %rd4, %r26;
@@ -7005,12 +8611,12 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
-; SM70-NEXT: @%p3 bra $L__BB164_1;
+; SM70-NEXT: @%p3 bra $L__BB219_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
-; SM70-NEXT: $L__BB164_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB219_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd13, %r27;
; SM70-NEXT: cvt.u64.u32 %rd14, %r28;
@@ -7030,12 +8636,12 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p6, %rd21, %rd16;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
-; SM70-NEXT: @%p6 bra $L__BB164_3;
+; SM70-NEXT: @%p6 bra $L__BB219_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
; SM70-NEXT: cvt.u32.u64 %r29, %rd22;
-; SM70-NEXT: $L__BB164_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB219_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd23, %r29;
; SM70-NEXT: cvt.u64.u32 %rd24, %r30;
@@ -7055,12 +8661,12 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p9, %rd31, %rd26;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
; SM70-NEXT: cvt.u32.u64 %r29, %rd31;
-; SM70-NEXT: @%p9 bra $L__BB164_5;
+; SM70-NEXT: @%p9 bra $L__BB219_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
; SM70-NEXT: cvt.u32.u64 %r31, %rd32;
-; SM70-NEXT: $L__BB164_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB219_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd33, %r31;
; SM70-NEXT: cvt.u64.u32 %rd34, %r32;
@@ -7080,7 +8686,7 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p12, %rd41, %rd36;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
; SM70-NEXT: cvt.u32.u64 %r31, %rd41;
-; SM70-NEXT: @%p12 bra $L__BB164_7;
+; SM70-NEXT: @%p12 bra $L__BB219_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
@@ -7090,6 +8696,34 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i32> %retval
}
+define <1 x i64> @usub_cond_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<7>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM70-NEXT: $L__BB220_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM70-NEXT: sub.s64 %rd4, %rd6, %rd1;
+; SM70-NEXT: selp.b64 %rd5, %rd4, %rd6, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM70-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
+; SM70-NEXT: mov.b64 %rd6, %rd2;
+; SM70-NEXT: @%p2 bra $L__BB220_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: usub_cond_acq_rel_v2i64_global_cta(
; SM70: {
@@ -7101,7 +8735,7 @@ define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
-; SM70-NEXT: $L__BB165_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB221_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
; SM70-NEXT: sub.s64 %rd6, %rd10, %rd1;
@@ -7109,10 +8743,10 @@ define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
; SM70-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
; SM70-NEXT: mov.b64 %rd10, %rd3;
-; SM70-NEXT: @%p2 bra $L__BB165_1;
+; SM70-NEXT: @%p2 bra $L__BB221_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
-; SM70-NEXT: $L__BB165_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB221_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
; SM70-NEXT: sub.s64 %rd8, %rd11, %rd2;
@@ -7120,7 +8754,7 @@ define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM70-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
; SM70-NEXT: mov.b64 %rd11, %rd4;
-; SM70-NEXT: @%p4 bra $L__BB165_3;
+; SM70-NEXT: @%p4 bra $L__BB221_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -7141,7 +8775,7 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
-; SM70-NEXT: $L__BB166_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB222_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd5, %rd18;
; SM70-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
@@ -7149,10 +8783,10 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: selp.b64 %rd11, %rd10, %rd5, %p1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
; SM70-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
-; SM70-NEXT: @%p2 bra $L__BB166_1;
+; SM70-NEXT: @%p2 bra $L__BB222_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
-; SM70-NEXT: $L__BB166_3: // %atomicrmw.start5
+; SM70-NEXT: $L__BB222_3: // %atomicrmw.start5
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd6, %rd19;
; SM70-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
@@ -7160,10 +8794,10 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: selp.b64 %rd13, %rd12, %rd6, %p3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
; SM70-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
-; SM70-NEXT: @%p4 bra $L__BB166_3;
+; SM70-NEXT: @%p4 bra $L__BB222_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end4
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
-; SM70-NEXT: $L__BB166_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB222_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
; SM70-NEXT: sub.s64 %rd14, %rd20, %rd1;
@@ -7171,10 +8805,10 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
; SM70-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
; SM70-NEXT: mov.b64 %rd20, %rd7;
-; SM70-NEXT: @%p6 bra $L__BB166_5;
+; SM70-NEXT: @%p6 bra $L__BB222_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
-; SM70-NEXT: $L__BB166_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB222_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
; SM70-NEXT: sub.s64 %rd16, %rd21, %rd2;
@@ -7182,7 +8816,7 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM70-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
; SM70-NEXT: mov.b64 %rd21, %rd8;
-; SM70-NEXT: @%p8 bra $L__BB166_7;
+; SM70-NEXT: @%p8 bra $L__BB222_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -7206,7 +8840,7 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
-; SM70-NEXT: $L__BB167_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB223_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd7, %rd34;
; SM70-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
@@ -7214,10 +8848,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd19, %rd18, %rd7, %p1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
; SM70-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
-; SM70-NEXT: @%p2 bra $L__BB167_1;
+; SM70-NEXT: @%p2 bra $L__BB223_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
-; SM70-NEXT: $L__BB167_3: // %atomicrmw.start9
+; SM70-NEXT: $L__BB223_3: // %atomicrmw.start9
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd8, %rd35;
; SM70-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
@@ -7225,10 +8859,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd21, %rd20, %rd8, %p3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
; SM70-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
-; SM70-NEXT: @%p4 bra $L__BB167_3;
+; SM70-NEXT: @%p4 bra $L__BB223_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end8
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
-; SM70-NEXT: $L__BB167_5: // %atomicrmw.start20
+; SM70-NEXT: $L__BB223_5: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd9, %rd36;
; SM70-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
@@ -7236,10 +8870,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd23, %rd22, %rd9, %p5;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
; SM70-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
-; SM70-NEXT: @%p6 bra $L__BB167_5;
+; SM70-NEXT: @%p6 bra $L__BB223_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end19
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
-; SM70-NEXT: $L__BB167_7: // %atomicrmw.start26
+; SM70-NEXT: $L__BB223_7: // %atomicrmw.start26
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd10, %rd37;
; SM70-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
@@ -7247,10 +8881,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd25, %rd24, %rd10, %p7;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
; SM70-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
-; SM70-NEXT: @%p8 bra $L__BB167_7;
+; SM70-NEXT: @%p8 bra $L__BB223_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end25
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
-; SM70-NEXT: $L__BB167_9: // %atomicrmw.start41
+; SM70-NEXT: $L__BB223_9: // %atomicrmw.start41
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd13, %rd38;
; SM70-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
@@ -7258,10 +8892,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd27, %rd26, %rd13, %p9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
; SM70-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
-; SM70-NEXT: @%p10 bra $L__BB167_9;
+; SM70-NEXT: @%p10 bra $L__BB223_9;
; SM70-NEXT: // %bb.10: // %atomicrmw.end40
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
-; SM70-NEXT: $L__BB167_11: // %atomicrmw.start47
+; SM70-NEXT: $L__BB223_11: // %atomicrmw.start47
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd14, %rd39;
; SM70-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
@@ -7269,10 +8903,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: selp.b64 %rd29, %rd28, %rd14, %p11;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
; SM70-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
-; SM70-NEXT: @%p12 bra $L__BB167_11;
+; SM70-NEXT: @%p12 bra $L__BB223_11;
; SM70-NEXT: // %bb.12: // %atomicrmw.end46
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
-; SM70-NEXT: $L__BB167_13: // %atomicrmw.start58
+; SM70-NEXT: $L__BB223_13: // %atomicrmw.start58
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
; SM70-NEXT: sub.s64 %rd30, %rd40, %rd11;
@@ -7280,10 +8914,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
; SM70-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
; SM70-NEXT: mov.b64 %rd40, %rd15;
-; SM70-NEXT: @%p14 bra $L__BB167_13;
+; SM70-NEXT: @%p14 bra $L__BB223_13;
; SM70-NEXT: // %bb.14: // %atomicrmw.end57
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
-; SM70-NEXT: $L__BB167_15: // %atomicrmw.start64
+; SM70-NEXT: $L__BB223_15: // %atomicrmw.start64
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
; SM70-NEXT: sub.s64 %rd32, %rd41, %rd12;
@@ -7291,7 +8925,7 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM70-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
; SM70-NEXT: mov.b64 %rd41, %rd16;
-; SM70-NEXT: @%p16 bra $L__BB167_15;
+; SM70-NEXT: @%p16 bra $L__BB223_15;
; SM70-NEXT: // %bb.16: // %atomicrmw.end63
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -7303,10 +8937,54 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i64> %retval
}
-define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM70-LABEL: usub_sat_acq_rel_v2i8_global_cta(
+define <1 x i8> @usub_sat_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v1i8_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<6>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [usub_sat_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB224_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: and.b16 %rs3, %rs2, 255;
+; SM70-NEXT: max.u16 %rs4, %rs3, %rs1;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB224_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
; SM70-NEXT: .reg .b16 %rs<17>;
; SM70-NEXT: .reg .b32 %r<22>;
; SM70-NEXT: .reg .b64 %rd<3>;
@@ -7325,7 +9003,7 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r21, [%rd1];
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r4;
-; SM70-NEXT: $L__BB168_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB225_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r21, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -7353,7 +9031,7 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r21, %r19;
; SM70-NEXT: setp.ne.b32 %p3, %r3, %r21;
; SM70-NEXT: mov.b32 %r21, %r3;
-; SM70-NEXT: @%p3 bra $L__BB168_1;
+; SM70-NEXT: @%p3 bra $L__BB225_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r20, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -7384,7 +9062,7 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
; SM70-NEXT: cvt.u16.u32 %rs9, %r10;
; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs13, %r13;
-; SM70-NEXT: $L__BB169_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB226_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -7412,7 +9090,7 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM70-NEXT: mov.b32 %r18, %r1;
-; SM70-NEXT: @%p1 bra $L__BB169_1;
+; SM70-NEXT: @%p1 bra $L__BB226_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -7446,7 +9124,7 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: cvt.u16.u32 %rs13, %r13;
; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM70-NEXT: $L__BB170_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB227_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -7512,7 +9190,7 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB170_1;
+; SM70-NEXT: @%p1 bra $L__BB227_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -7521,6 +9199,49 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
ret <8 x i8> %retval
}
+define <1 x i16> @usub_sat_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB228_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM70-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB228_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: usub_sat_acq_rel_v2i16_global_cta(
; SM70: {
@@ -7535,7 +9256,7 @@ define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NEXT: $L__BB171_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB229_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r7;
; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -7551,7 +9272,7 @@ define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
; SM70-NEXT: setp.ne.b32 %p3, %r1, %r7;
; SM70-NEXT: mov.b32 %r7, %r1;
-; SM70-NEXT: @%p3 bra $L__BB171_1;
+; SM70-NEXT: @%p3 bra $L__BB229_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -7577,7 +9298,7 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: cvt.u32.u64 %r11, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
-; SM70-NEXT: $L__BB172_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB230_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r11;
; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -7613,7 +9334,7 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB172_1;
+; SM70-NEXT: @%p5 bra $L__BB230_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r11, %r12};
@@ -7639,7 +9360,7 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: cvt.u32.u64 %r21, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
-; SM70-NEXT: $L__BB173_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB231_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r21;
; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -7675,12 +9396,12 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB173_1;
+; SM70-NEXT: @%p5 bra $L__BB231_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd12;
-; SM70-NEXT: $L__BB173_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB231_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r3;
; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r23;
@@ -7718,7 +9439,7 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd21;
-; SM70-NEXT: @%p10 bra $L__BB173_3;
+; SM70-NEXT: @%p10 bra $L__BB231_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
@@ -7727,6 +9448,34 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i16> %retval
}
+define <1 x i32> @usub_sat_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [usub_sat_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB232_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r3, %r5, %r1;
+; SM70-NEXT: sub.s32 %r4, %r3, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p1 bra $L__BB232_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: usub_sat_acq_rel_v2i32_global_cta(
; SM70: {
@@ -7741,7 +9490,7 @@ define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: $L__BB174_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB233_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.u32 %r3, %r7, %r1;
; SM70-NEXT: sub.s32 %r4, %r3, %r1;
@@ -7759,7 +9508,7 @@ define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB174_1;
+; SM70-NEXT: @%p1 bra $L__BB233_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -7782,7 +9531,7 @@ define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
-; SM70-NEXT: $L__BB175_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB234_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.u32 %r5, %r13, %r1;
; SM70-NEXT: sub.s32 %r6, %r5, %r1;
@@ -7800,12 +9549,12 @@ define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB175_1;
+; SM70-NEXT: @%p1 bra $L__BB234_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r15, %rd12;
-; SM70-NEXT: $L__BB175_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB234_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.u32 %r9, %r15, %r3;
; SM70-NEXT: sub.s32 %r10, %r9, %r3;
@@ -7823,7 +9572,7 @@ define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r15, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB175_3;
+; SM70-NEXT: @%p2 bra $L__BB234_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
@@ -7847,7 +9596,7 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
-; SM70-NEXT: $L__BB176_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB235_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.u32 %r9, %r25, %r5;
; SM70-NEXT: sub.s32 %r10, %r9, %r5;
@@ -7865,12 +9614,12 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB176_1;
+; SM70-NEXT: @%p1 bra $L__BB235_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
-; SM70-NEXT: $L__BB176_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB235_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.u32 %r13, %r27, %r7;
; SM70-NEXT: sub.s32 %r14, %r13, %r7;
@@ -7888,12 +9637,12 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB176_3;
+; SM70-NEXT: @%p2 bra $L__BB235_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
; SM70-NEXT: cvt.u32.u64 %r29, %rd22;
-; SM70-NEXT: $L__BB176_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB235_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.u32 %r17, %r29, %r1;
; SM70-NEXT: sub.s32 %r18, %r17, %r1;
@@ -7911,12 +9660,12 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
; SM70-NEXT: cvt.u32.u64 %r29, %rd31;
-; SM70-NEXT: @%p3 bra $L__BB176_5;
+; SM70-NEXT: @%p3 bra $L__BB235_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
; SM70-NEXT: cvt.u32.u64 %r31, %rd32;
-; SM70-NEXT: $L__BB176_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB235_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.u32 %r21, %r31, %r3;
; SM70-NEXT: sub.s32 %r22, %r21, %r3;
@@ -7934,7 +9683,7 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
; SM70-NEXT: cvt.u32.u64 %r31, %rd41;
-; SM70-NEXT: @%p4 bra $L__BB176_7;
+; SM70-NEXT: @%p4 bra $L__BB235_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
@@ -7944,6 +9693,33 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i32> %retval
}
+define <1 x i64> @usub_sat_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b64 %rd<7>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM70-NEXT: $L__BB236_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd4, %rd6, %rd1;
+; SM70-NEXT: sub.s64 %rd5, %rd4, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
+; SM70-NEXT: mov.b64 %rd6, %rd2;
+; SM70-NEXT: @%p1 bra $L__BB236_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: usub_sat_acq_rel_v2i64_global_cta(
; SM70: {
@@ -7955,24 +9731,24 @@ define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
-; SM70-NEXT: $L__BB177_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB237_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.u64 %rd6, %rd10, %rd1;
; SM70-NEXT: sub.s64 %rd7, %rd6, %rd1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
; SM70-NEXT: mov.b64 %rd10, %rd3;
-; SM70-NEXT: @%p1 bra $L__BB177_1;
+; SM70-NEXT: @%p1 bra $L__BB237_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
-; SM70-NEXT: $L__BB177_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB237_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.u64 %rd8, %rd11, %rd2;
; SM70-NEXT: sub.s64 %rd9, %rd8, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
; SM70-NEXT: mov.b64 %rd11, %rd4;
-; SM70-NEXT: @%p2 bra $L__BB177_3;
+; SM70-NEXT: @%p2 bra $L__BB237_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -7993,44 +9769,44 @@ define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
-; SM70-NEXT: $L__BB178_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB238_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd5, %rd18;
; SM70-NEXT: max.u64 %rd10, %rd5, %rd3;
; SM70-NEXT: sub.s64 %rd11, %rd10, %rd3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
; SM70-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
-; SM70-NEXT: @%p1 bra $L__BB178_1;
+; SM70-NEXT: @%p1 bra $L__BB238_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
-; SM70-NEXT: $L__BB178_3: // %atomicrmw.start5
+; SM70-NEXT: $L__BB238_3: // %atomicrmw.start5
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd6, %rd19;
; SM70-NEXT: max.u64 %rd12, %rd6, %rd4;
; SM70-NEXT: sub.s64 %rd13, %rd12, %rd4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
; SM70-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
-; SM70-NEXT: @%p2 bra $L__BB178_3;
+; SM70-NEXT: @%p2 bra $L__BB238_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end4
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
-; SM70-NEXT: $L__BB178_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB238_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.u64 %rd14, %rd20, %rd1;
; SM70-NEXT: sub.s64 %rd15, %rd14, %rd1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
; SM70-NEXT: mov.b64 %rd20, %rd7;
-; SM70-NEXT: @%p3 bra $L__BB178_5;
+; SM70-NEXT: @%p3 bra $L__BB238_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
-; SM70-NEXT: $L__BB178_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB238_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.u64 %rd16, %rd21, %rd2;
; SM70-NEXT: sub.s64 %rd17, %rd16, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
; SM70-NEXT: mov.b64 %rd21, %rd8;
-; SM70-NEXT: @%p4 bra $L__BB178_7;
+; SM70-NEXT: @%p4 bra $L__BB238_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -8054,84 +9830,84 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
-; SM70-NEXT: $L__BB179_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB239_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd7, %rd34;
; SM70-NEXT: max.u64 %rd18, %rd7, %rd5;
; SM70-NEXT: sub.s64 %rd19, %rd18, %rd5;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
; SM70-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
-; SM70-NEXT: @%p1 bra $L__BB179_1;
+; SM70-NEXT: @%p1 bra $L__BB239_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
-; SM70-NEXT: $L__BB179_3: // %atomicrmw.start9
+; SM70-NEXT: $L__BB239_3: // %atomicrmw.start9
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd8, %rd35;
; SM70-NEXT: max.u64 %rd20, %rd8, %rd6;
; SM70-NEXT: sub.s64 %rd21, %rd20, %rd6;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
; SM70-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
-; SM70-NEXT: @%p2 bra $L__BB179_3;
+; SM70-NEXT: @%p2 bra $L__BB239_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end8
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
-; SM70-NEXT: $L__BB179_5: // %atomicrmw.start20
+; SM70-NEXT: $L__BB239_5: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd9, %rd36;
; SM70-NEXT: max.u64 %rd22, %rd9, %rd3;
; SM70-NEXT: sub.s64 %rd23, %rd22, %rd3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
; SM70-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
-; SM70-NEXT: @%p3 bra $L__BB179_5;
+; SM70-NEXT: @%p3 bra $L__BB239_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end19
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
-; SM70-NEXT: $L__BB179_7: // %atomicrmw.start26
+; SM70-NEXT: $L__BB239_7: // %atomicrmw.start26
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd10, %rd37;
; SM70-NEXT: max.u64 %rd24, %rd10, %rd4;
; SM70-NEXT: sub.s64 %rd25, %rd24, %rd4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
; SM70-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
-; SM70-NEXT: @%p4 bra $L__BB179_7;
+; SM70-NEXT: @%p4 bra $L__BB239_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end25
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
-; SM70-NEXT: $L__BB179_9: // %atomicrmw.start41
+; SM70-NEXT: $L__BB239_9: // %atomicrmw.start41
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd13, %rd38;
; SM70-NEXT: max.u64 %rd26, %rd13, %rd1;
; SM70-NEXT: sub.s64 %rd27, %rd26, %rd1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
; SM70-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
-; SM70-NEXT: @%p5 bra $L__BB179_9;
+; SM70-NEXT: @%p5 bra $L__BB239_9;
; SM70-NEXT: // %bb.10: // %atomicrmw.end40
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
-; SM70-NEXT: $L__BB179_11: // %atomicrmw.start47
+; SM70-NEXT: $L__BB239_11: // %atomicrmw.start47
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd14, %rd39;
; SM70-NEXT: max.u64 %rd28, %rd14, %rd2;
; SM70-NEXT: sub.s64 %rd29, %rd28, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
; SM70-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
-; SM70-NEXT: @%p6 bra $L__BB179_11;
+; SM70-NEXT: @%p6 bra $L__BB239_11;
; SM70-NEXT: // %bb.12: // %atomicrmw.end46
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
-; SM70-NEXT: $L__BB179_13: // %atomicrmw.start58
+; SM70-NEXT: $L__BB239_13: // %atomicrmw.start58
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.u64 %rd30, %rd40, %rd11;
; SM70-NEXT: sub.s64 %rd31, %rd30, %rd11;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
; SM70-NEXT: mov.b64 %rd40, %rd15;
-; SM70-NEXT: @%p7 bra $L__BB179_13;
+; SM70-NEXT: @%p7 bra $L__BB239_13;
; SM70-NEXT: // %bb.14: // %atomicrmw.end57
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
-; SM70-NEXT: $L__BB179_15: // %atomicrmw.start64
+; SM70-NEXT: $L__BB239_15: // %atomicrmw.start64
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.u64 %rd32, %rd41, %rd12;
; SM70-NEXT: sub.s64 %rd33, %rd32, %rd12;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
; SM70-NEXT: mov.b64 %rd41, %rd16;
-; SM70-NEXT: @%p8 bra $L__BB179_15;
+; SM70-NEXT: @%p8 bra $L__BB239_15;
; SM70-NEXT: // %bb.16: // %atomicrmw.end63
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -8143,6 +9919,24 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i64> %retval
}
+define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM70-LABEL: fadd_acq_rel_v1f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-LABEL: fadd_acq_rel_v2f32_global_cta(
; SM70: {
@@ -8210,6 +10004,33 @@ define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
ret <8 x float> %retval
}
+define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM70-LABEL: fsub_acq_rel_v1f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB244_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r3, %r4, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM70-NEXT: mov.b32 %r4, %r2;
+; SM70-NEXT: @%p1 bra $L__BB244_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-LABEL: fsub_acq_rel_v2f32_global_cta(
; SM70: {
@@ -8224,7 +10045,7 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB183_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB245_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f32 %r3, %r5, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -8240,7 +10061,7 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB183_1;
+; SM70-NEXT: @%p1 bra $L__BB245_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -8263,7 +10084,7 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB184_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB246_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f32 %r5, %r9, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -8279,12 +10100,12 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB184_1;
+; SM70-NEXT: @%p1 bra $L__BB246_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM70-NEXT: $L__BB184_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB246_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f32 %r7, %r11, %r3;
; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -8300,7 +10121,7 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB184_3;
+; SM70-NEXT: @%p2 bra $L__BB246_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -8324,7 +10145,7 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM70-NEXT: $L__BB185_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB247_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f32 %r9, %r17, %r5;
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
@@ -8340,12 +10161,12 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB185_1;
+; SM70-NEXT: @%p1 bra $L__BB247_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd12;
-; SM70-NEXT: $L__BB185_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB247_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f32 %r11, %r19, %r7;
; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
@@ -8361,12 +10182,12 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB185_3;
+; SM70-NEXT: @%p2 bra $L__BB247_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd22;
-; SM70-NEXT: $L__BB185_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB247_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f32 %r13, %r21, %r1;
; SM70-NEXT: cvt.u64.u32 %rd23, %r13;
@@ -8382,12 +10203,12 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd31;
-; SM70-NEXT: @%p3 bra $L__BB185_5;
+; SM70-NEXT: @%p3 bra $L__BB247_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd32;
-; SM70-NEXT: $L__BB185_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB247_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f32 %r15, %r23, %r3;
; SM70-NEXT: cvt.u64.u32 %rd33, %r15;
@@ -8403,7 +10224,7 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd41;
-; SM70-NEXT: @%p4 bra $L__BB185_7;
+; SM70-NEXT: @%p4 bra $L__BB247_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -8413,6 +10234,33 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
ret <8 x float> %retval
}
+define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM70-LABEL: fmin_acq_rel_v1f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB248_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r3, %r4, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM70-NEXT: mov.b32 %r4, %r2;
+; SM70-NEXT: @%p1 bra $L__BB248_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-LABEL: fmin_acq_rel_v2f32_global_cta(
; SM70: {
@@ -8427,7 +10275,7 @@ define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB186_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB249_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: min.f32 %r3, %r5, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -8443,7 +10291,7 @@ define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB186_1;
+; SM70-NEXT: @%p1 bra $L__BB249_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -8466,7 +10314,7 @@ define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB187_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB250_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: min.f32 %r5, %r9, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -8482,12 +10330,12 @@ define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB187_1;
+; SM70-NEXT: @%p1 bra $L__BB250_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM70-NEXT: $L__BB187_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB250_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: min.f32 %r7, %r11, %r3;
; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -8503,7 +10351,7 @@ define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB187_3;
+; SM70-NEXT: @%p2 bra $L__BB250_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -8527,7 +10375,7 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM70-NEXT: $L__BB188_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB251_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: min.f32 %r9, %r17, %r5;
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
@@ -8543,12 +10391,12 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB188_1;
+; SM70-NEXT: @%p1 bra $L__BB251_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd12;
-; SM70-NEXT: $L__BB188_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB251_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: min.f32 %r11, %r19, %r7;
; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
@@ -8564,12 +10412,12 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB188_3;
+; SM70-NEXT: @%p2 bra $L__BB251_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd22;
-; SM70-NEXT: $L__BB188_5: // %atomicrmw.start15
+; SM70-NEXT: $L__BB251_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: min.f32 %r13, %r21, %r1;
; SM70-NEXT: cvt.u64.u32 %rd23, %r13;
@@ -8585,12 +10433,12 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd31;
-; SM70-NEXT: @%p3 bra $L__BB188_5;
+; SM70-NEXT: @%p3 bra $L__BB251_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd32;
-; SM70-NEXT: $L__BB188_7: // %atomicrmw.start20
+; SM70-NEXT: $L__BB251_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: min.f32 %r15, %r23, %r3;
; SM70-NEXT: cvt.u64.u32 %rd33, %r15;
@@ -8606,7 +10454,7 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd41;
-; SM70-NEXT: @%p4 bra $L__BB188_7;
+; SM70-NEXT: @%p4 bra $L__BB251_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -8616,6 +10464,33 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
ret <8 x float> %retval
}
+define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM70-LABEL: fmax_acq_rel_v1f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB252_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r3, %r4, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM70-NEXT: mov.b32 %r4, %r2;
+; SM70-NEXT: @%p1 bra $L__BB252_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-LABEL: fmax_acq_rel_v2f32_global_cta(
; SM70: {
@@ -8630,7 +10505,7 @@ define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB189_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB253_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.f32 %r3, %r5, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -8646,7 +10521,7 @@ define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB189_1;
+; SM70-NEXT: @%p1 bra $L__BB253_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -8669,7 +10544,7 @@ define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB190_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB254_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.f32 %r5, %r9, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -8685,12 +10560,12 @@ define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB190_1;
+; SM70-NEXT: @%p1 bra $L__BB254_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM70-NEXT: $L__BB190_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB254_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.f32 %r7, %r11, %r3;
; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -8706,7 +10581,7 @@ define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB190_3;
+; SM70-NEXT: @%p2 bra $L__BB254_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -8730,7 +10605,7 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM70-NEXT: $L__BB191_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB255_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.f32 %r9, %r17, %r5;
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
@@ -8746,12 +10621,12 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB191_1;
+; SM70-NEXT: @%p1 bra $L__BB255_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd12;
-; SM70-NEXT: $L__BB191_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB255_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.f32 %r11, %r19, %r7;
; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
@@ -8767,12 +10642,12 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB191_3;
+; SM70-NEXT: @%p2 bra $L__BB255_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd22;
-; SM70-NEXT: $L__BB191_5: // %atomicrmw.start15
+; SM70-NEXT: $L__BB255_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.f32 %r13, %r21, %r1;
; SM70-NEXT: cvt.u64.u32 %rd23, %r13;
@@ -8788,12 +10663,12 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd31;
-; SM70-NEXT: @%p3 bra $L__BB191_5;
+; SM70-NEXT: @%p3 bra $L__BB255_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd32;
-; SM70-NEXT: $L__BB191_7: // %atomicrmw.start20
+; SM70-NEXT: $L__BB255_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.f32 %r15, %r23, %r3;
; SM70-NEXT: cvt.u64.u32 %rd33, %r15;
@@ -8809,7 +10684,7 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM70-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd41;
-; SM70-NEXT: @%p4 bra $L__BB191_7;
+; SM70-NEXT: @%p4 bra $L__BB255_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -8819,6 +10694,41 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
ret <8 x float> %retval
}
+define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM70-LABEL: fminimum_acq_rel_v1f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1];
+; SM70-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM70-NEXT: $L__BB256_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r8, %r1;
+; SM70-NEXT: min.f32 %r3, %r8, %r1;
+; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r8, -2147483648;
+; SM70-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM70-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r8, %r7;
+; SM70-NEXT: setp.ne.b32 %p5, %r2, %r8;
+; SM70-NEXT: mov.b32 %r8, %r2;
+; SM70-NEXT: @%p5 bra $L__BB256_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-LABEL: fminimum_acq_rel_v2f32_global_cta(
; SM70: {
@@ -8835,7 +10745,7 @@ define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
; SM70-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
; SM70-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
-; SM70-NEXT: $L__BB192_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB257_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p1, %r13, %r1;
; SM70-NEXT: min.f32 %r3, %r13, %r1;
@@ -8865,7 +10775,7 @@ define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB192_1;
+; SM70-NEXT: @%p9 bra $L__BB257_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
@@ -8890,7 +10800,7 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
; SM70-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
; SM70-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
-; SM70-NEXT: $L__BB193_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB258_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p1, %r25, %r1;
; SM70-NEXT: min.f32 %r5, %r25, %r1;
@@ -8920,14 +10830,14 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB193_1;
+; SM70-NEXT: @%p9 bra $L__BB258_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
; SM70-NEXT: setp.eq.b32 %p12, %r3, -2147483648;
; SM70-NEXT: setp.eq.b32 %p16, %r4, -2147483648;
-; SM70-NEXT: $L__BB193_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB258_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p10, %r27, %r3;
; SM70-NEXT: min.f32 %r15, %r27, %r3;
@@ -8957,7 +10867,7 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
-; SM70-NEXT: @%p18 bra $L__BB193_3;
+; SM70-NEXT: @%p18 bra $L__BB258_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
@@ -8983,7 +10893,7 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
; SM70-NEXT: setp.eq.b32 %p3, %r5, -2147483648;
; SM70-NEXT: setp.eq.b32 %p7, %r6, -2147483648;
-; SM70-NEXT: $L__BB194_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB259_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p1, %r49, %r5;
; SM70-NEXT: min.f32 %r9, %r49, %r5;
@@ -9013,14 +10923,14 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB194_1;
+; SM70-NEXT: @%p9 bra $L__BB259_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r51, %rd12;
; SM70-NEXT: setp.eq.b32 %p12, %r7, -2147483648;
; SM70-NEXT: setp.eq.b32 %p16, %r8, -2147483648;
-; SM70-NEXT: $L__BB194_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB259_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p10, %r51, %r7;
; SM70-NEXT: min.f32 %r19, %r51, %r7;
@@ -9050,14 +10960,14 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r51, %rd21;
-; SM70-NEXT: @%p18 bra $L__BB194_3;
+; SM70-NEXT: @%p18 bra $L__BB259_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
; SM70-NEXT: cvt.u32.u64 %r53, %rd22;
; SM70-NEXT: setp.eq.b32 %p21, %r1, -2147483648;
; SM70-NEXT: setp.eq.b32 %p25, %r2, -2147483648;
-; SM70-NEXT: $L__BB194_5: // %atomicrmw.start15
+; SM70-NEXT: $L__BB259_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p19, %r53, %r1;
; SM70-NEXT: min.f32 %r29, %r53, %r1;
@@ -9087,14 +10997,14 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
; SM70-NEXT: cvt.u32.u64 %r53, %rd31;
-; SM70-NEXT: @%p27 bra $L__BB194_5;
+; SM70-NEXT: @%p27 bra $L__BB259_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
; SM70-NEXT: cvt.u32.u64 %r55, %rd32;
; SM70-NEXT: setp.eq.b32 %p30, %r3, -2147483648;
; SM70-NEXT: setp.eq.b32 %p34, %r4, -2147483648;
-; SM70-NEXT: $L__BB194_7: // %atomicrmw.start20
+; SM70-NEXT: $L__BB259_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p28, %r55, %r3;
; SM70-NEXT: min.f32 %r39, %r55, %r3;
@@ -9124,7 +11034,7 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
; SM70-NEXT: cvt.u32.u64 %r55, %rd41;
-; SM70-NEXT: @%p36 bra $L__BB194_7;
+; SM70-NEXT: @%p36 bra $L__BB259_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
@@ -9134,6 +11044,41 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x float> %retval
}
+define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v1f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1];
+; SM70-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM70-NEXT: $L__BB260_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r8, %r1;
+; SM70-NEXT: max.f32 %r3, %r8, %r1;
+; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r8, 0;
+; SM70-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM70-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r8, %r7;
+; SM70-NEXT: setp.ne.b32 %p5, %r2, %r8;
+; SM70-NEXT: mov.b32 %r8, %r2;
+; SM70-NEXT: @%p5 bra $L__BB260_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-LABEL: fmaximum_acq_rel_v2f32_global_cta(
; SM70: {
@@ -9150,7 +11095,7 @@ define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
; SM70-NEXT: setp.eq.b32 %p3, %r1, 0;
; SM70-NEXT: setp.eq.b32 %p7, %r2, 0;
-; SM70-NEXT: $L__BB195_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB261_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p1, %r13, %r1;
; SM70-NEXT: max.f32 %r3, %r13, %r1;
@@ -9180,7 +11125,7 @@ define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB195_1;
+; SM70-NEXT: @%p9 bra $L__BB261_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
@@ -9205,7 +11150,7 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
; SM70-NEXT: setp.eq.b32 %p3, %r1, 0;
; SM70-NEXT: setp.eq.b32 %p7, %r2, 0;
-; SM70-NEXT: $L__BB196_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB262_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p1, %r25, %r1;
; SM70-NEXT: max.f32 %r5, %r25, %r1;
@@ -9235,14 +11180,14 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB196_1;
+; SM70-NEXT: @%p9 bra $L__BB262_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
; SM70-NEXT: setp.eq.b32 %p12, %r3, 0;
; SM70-NEXT: setp.eq.b32 %p16, %r4, 0;
-; SM70-NEXT: $L__BB196_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB262_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p10, %r27, %r3;
; SM70-NEXT: max.f32 %r15, %r27, %r3;
@@ -9272,7 +11217,7 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
-; SM70-NEXT: @%p18 bra $L__BB196_3;
+; SM70-NEXT: @%p18 bra $L__BB262_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
@@ -9298,7 +11243,7 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
; SM70-NEXT: setp.eq.b32 %p3, %r5, 0;
; SM70-NEXT: setp.eq.b32 %p7, %r6, 0;
-; SM70-NEXT: $L__BB197_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB263_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p1, %r49, %r5;
; SM70-NEXT: max.f32 %r9, %r49, %r5;
@@ -9328,14 +11273,14 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB197_1;
+; SM70-NEXT: @%p9 bra $L__BB263_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r51, %rd12;
; SM70-NEXT: setp.eq.b32 %p12, %r7, 0;
; SM70-NEXT: setp.eq.b32 %p16, %r8, 0;
-; SM70-NEXT: $L__BB197_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB263_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p10, %r51, %r7;
; SM70-NEXT: max.f32 %r19, %r51, %r7;
@@ -9365,14 +11310,14 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r51, %rd21;
-; SM70-NEXT: @%p18 bra $L__BB197_3;
+; SM70-NEXT: @%p18 bra $L__BB263_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
; SM70-NEXT: cvt.u32.u64 %r53, %rd22;
; SM70-NEXT: setp.eq.b32 %p21, %r1, 0;
; SM70-NEXT: setp.eq.b32 %p25, %r2, 0;
-; SM70-NEXT: $L__BB197_5: // %atomicrmw.start15
+; SM70-NEXT: $L__BB263_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p19, %r53, %r1;
; SM70-NEXT: max.f32 %r29, %r53, %r1;
@@ -9402,14 +11347,14 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
; SM70-NEXT: cvt.u32.u64 %r53, %rd31;
-; SM70-NEXT: @%p27 bra $L__BB197_5;
+; SM70-NEXT: @%p27 bra $L__BB263_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
; SM70-NEXT: cvt.u32.u64 %r55, %rd32;
; SM70-NEXT: setp.eq.b32 %p30, %r3, 0;
; SM70-NEXT: setp.eq.b32 %p34, %r4, 0;
-; SM70-NEXT: $L__BB197_7: // %atomicrmw.start20
+; SM70-NEXT: $L__BB263_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f32 %p28, %r55, %r3;
; SM70-NEXT: max.f32 %r39, %r55, %r3;
@@ -9439,7 +11384,7 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
; SM70-NEXT: cvt.u32.u64 %r55, %rd41;
-; SM70-NEXT: @%p36 bra $L__BB197_7;
+; SM70-NEXT: @%p36 bra $L__BB263_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
@@ -9449,6 +11394,23 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x float> %retval
}
+define <1 x double> @fadd_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM70-LABEL: fadd_acq_rel_v1f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fadd_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM70-LABEL: fadd_acq_rel_v2f64_global_cta(
; SM70: {
@@ -9519,6 +11481,32 @@ define <8 x double> @fadd_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x double> %retval
}
+define <1 x double> @fsub_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM70-LABEL: fsub_acq_rel_v1f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [fsub_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM70-NEXT: $L__BB268_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd4, %rd5, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM70-NEXT: mov.b64 %rd5, %rd2;
+; SM70-NEXT: @%p1 bra $L__BB268_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM70-LABEL: fsub_acq_rel_v2f64_global_cta(
; SM70: {
@@ -9530,22 +11518,22 @@ define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd5];
-; SM70-NEXT: $L__BB201_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB269_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f64 %rd6, %rd8, %rd1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
; SM70-NEXT: mov.b64 %rd8, %rd3;
-; SM70-NEXT: @%p1 bra $L__BB201_1;
+; SM70-NEXT: @%p1 bra $L__BB269_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd5+8];
-; SM70-NEXT: $L__BB201_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB269_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f64 %rd7, %rd9, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
; SM70-NEXT: mov.b64 %rd9, %rd4;
-; SM70-NEXT: @%p2 bra $L__BB201_3;
+; SM70-NEXT: @%p2 bra $L__BB269_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -9566,40 +11554,40 @@ define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd9];
-; SM70-NEXT: $L__BB202_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB270_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd5, %rd14;
; SM70-NEXT: sub.rn.f64 %rd10, %rd5, %rd3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
; SM70-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
-; SM70-NEXT: @%p1 bra $L__BB202_1;
+; SM70-NEXT: @%p1 bra $L__BB270_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd9+8];
-; SM70-NEXT: $L__BB202_3: // %atomicrmw.start5
+; SM70-NEXT: $L__BB270_3: // %atomicrmw.start5
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd6, %rd15;
; SM70-NEXT: sub.rn.f64 %rd11, %rd6, %rd4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
; SM70-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
-; SM70-NEXT: @%p2 bra $L__BB202_3;
+; SM70-NEXT: @%p2 bra $L__BB270_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end4
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd9+16];
-; SM70-NEXT: $L__BB202_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB270_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f64 %rd12, %rd16, %rd1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
; SM70-NEXT: mov.b64 %rd16, %rd7;
-; SM70-NEXT: @%p3 bra $L__BB202_5;
+; SM70-NEXT: @%p3 bra $L__BB270_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd9+24];
-; SM70-NEXT: $L__BB202_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB270_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f64 %rd13, %rd17, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
; SM70-NEXT: mov.b64 %rd17, %rd8;
-; SM70-NEXT: @%p4 bra $L__BB202_7;
+; SM70-NEXT: @%p4 bra $L__BB270_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -9623,76 +11611,76 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fsub_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd17];
-; SM70-NEXT: $L__BB203_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB271_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd7, %rd26;
; SM70-NEXT: sub.rn.f64 %rd18, %rd7, %rd5;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
; SM70-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
-; SM70-NEXT: @%p1 bra $L__BB203_1;
+; SM70-NEXT: @%p1 bra $L__BB271_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd17+8];
-; SM70-NEXT: $L__BB203_3: // %atomicrmw.start9
+; SM70-NEXT: $L__BB271_3: // %atomicrmw.start9
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd8, %rd27;
; SM70-NEXT: sub.rn.f64 %rd19, %rd8, %rd6;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
; SM70-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
-; SM70-NEXT: @%p2 bra $L__BB203_3;
+; SM70-NEXT: @%p2 bra $L__BB271_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end8
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd17+16];
-; SM70-NEXT: $L__BB203_5: // %atomicrmw.start20
+; SM70-NEXT: $L__BB271_5: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd9, %rd28;
; SM70-NEXT: sub.rn.f64 %rd20, %rd9, %rd3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
; SM70-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
-; SM70-NEXT: @%p3 bra $L__BB203_5;
+; SM70-NEXT: @%p3 bra $L__BB271_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end19
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd17+24];
-; SM70-NEXT: $L__BB203_7: // %atomicrmw.start26
+; SM70-NEXT: $L__BB271_7: // %atomicrmw.start26
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd10, %rd29;
; SM70-NEXT: sub.rn.f64 %rd21, %rd10, %rd4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
; SM70-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
-; SM70-NEXT: @%p4 bra $L__BB203_7;
+; SM70-NEXT: @%p4 bra $L__BB271_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end25
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd17+32];
-; SM70-NEXT: $L__BB203_9: // %atomicrmw.start41
+; SM70-NEXT: $L__BB271_9: // %atomicrmw.start41
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd13, %rd30;
; SM70-NEXT: sub.rn.f64 %rd22, %rd13, %rd1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
-; SM70-NEXT: @%p5 bra $L__BB203_9;
+; SM70-NEXT: @%p5 bra $L__BB271_9;
; SM70-NEXT: // %bb.10: // %atomicrmw.end40
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd17+40];
-; SM70-NEXT: $L__BB203_11: // %atomicrmw.start47
+; SM70-NEXT: $L__BB271_11: // %atomicrmw.start47
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd14, %rd31;
; SM70-NEXT: sub.rn.f64 %rd23, %rd14, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
; SM70-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
-; SM70-NEXT: @%p6 bra $L__BB203_11;
+; SM70-NEXT: @%p6 bra $L__BB271_11;
; SM70-NEXT: // %bb.12: // %atomicrmw.end46
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd17+48];
-; SM70-NEXT: $L__BB203_13: // %atomicrmw.start58
+; SM70-NEXT: $L__BB271_13: // %atomicrmw.start58
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f64 %rd24, %rd32, %rd11;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
; SM70-NEXT: mov.b64 %rd32, %rd15;
-; SM70-NEXT: @%p7 bra $L__BB203_13;
+; SM70-NEXT: @%p7 bra $L__BB271_13;
; SM70-NEXT: // %bb.14: // %atomicrmw.end57
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd17+56];
-; SM70-NEXT: $L__BB203_15: // %atomicrmw.start64
+; SM70-NEXT: $L__BB271_15: // %atomicrmw.start64
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f64 %rd25, %rd33, %rd12;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
; SM70-NEXT: mov.b64 %rd33, %rd16;
-; SM70-NEXT: @%p8 bra $L__BB203_15;
+; SM70-NEXT: @%p8 bra $L__BB271_15;
; SM70-NEXT: // %bb.16: // %atomicrmw.end63
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -9704,6 +11692,32 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x double> %retval
}
+define <1 x double> @fmin_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM70-LABEL: fmin_acq_rel_v1f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM70-NEXT: $L__BB272_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd4, %rd5, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM70-NEXT: mov.b64 %rd5, %rd2;
+; SM70-NEXT: @%p1 bra $L__BB272_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM70-LABEL: fmin_acq_rel_v2f64_global_cta(
; SM70: {
@@ -9715,22 +11729,22 @@ define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd5];
-; SM70-NEXT: $L__BB204_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB273_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: min.f64 %rd6, %rd8, %rd1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
; SM70-NEXT: mov.b64 %rd8, %rd3;
-; SM70-NEXT: @%p1 bra $L__BB204_1;
+; SM70-NEXT: @%p1 bra $L__BB273_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd5+8];
-; SM70-NEXT: $L__BB204_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB273_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: min.f64 %rd7, %rd9, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
; SM70-NEXT: mov.b64 %rd9, %rd4;
-; SM70-NEXT: @%p2 bra $L__BB204_3;
+; SM70-NEXT: @%p2 bra $L__BB273_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -9751,40 +11765,40 @@ define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd9];
-; SM70-NEXT: $L__BB205_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB274_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd5, %rd14;
; SM70-NEXT: min.f64 %rd10, %rd5, %rd3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
; SM70-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
-; SM70-NEXT: @%p1 bra $L__BB205_1;
+; SM70-NEXT: @%p1 bra $L__BB274_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd9+8];
-; SM70-NEXT: $L__BB205_3: // %atomicrmw.start5
+; SM70-NEXT: $L__BB274_3: // %atomicrmw.start5
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd6, %rd15;
; SM70-NEXT: min.f64 %rd11, %rd6, %rd4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
; SM70-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
-; SM70-NEXT: @%p2 bra $L__BB205_3;
+; SM70-NEXT: @%p2 bra $L__BB274_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end4
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd9+16];
-; SM70-NEXT: $L__BB205_5: // %atomicrmw.start15
+; SM70-NEXT: $L__BB274_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: min.f64 %rd12, %rd16, %rd1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
; SM70-NEXT: mov.b64 %rd16, %rd7;
-; SM70-NEXT: @%p3 bra $L__BB205_5;
+; SM70-NEXT: @%p3 bra $L__BB274_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd9+24];
-; SM70-NEXT: $L__BB205_7: // %atomicrmw.start20
+; SM70-NEXT: $L__BB274_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: min.f64 %rd13, %rd17, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
; SM70-NEXT: mov.b64 %rd17, %rd8;
-; SM70-NEXT: @%p4 bra $L__BB205_7;
+; SM70-NEXT: @%p4 bra $L__BB274_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -9808,76 +11822,76 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmin_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd17];
-; SM70-NEXT: $L__BB206_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB275_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd7, %rd26;
; SM70-NEXT: min.f64 %rd18, %rd7, %rd5;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
; SM70-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
-; SM70-NEXT: @%p1 bra $L__BB206_1;
+; SM70-NEXT: @%p1 bra $L__BB275_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd17+8];
-; SM70-NEXT: $L__BB206_3: // %atomicrmw.start9
+; SM70-NEXT: $L__BB275_3: // %atomicrmw.start9
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd8, %rd27;
; SM70-NEXT: min.f64 %rd19, %rd8, %rd6;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
; SM70-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
-; SM70-NEXT: @%p2 bra $L__BB206_3;
+; SM70-NEXT: @%p2 bra $L__BB275_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end8
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd17+16];
-; SM70-NEXT: $L__BB206_5: // %atomicrmw.start19
+; SM70-NEXT: $L__BB275_5: // %atomicrmw.start19
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd9, %rd28;
; SM70-NEXT: min.f64 %rd20, %rd9, %rd3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
; SM70-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
-; SM70-NEXT: @%p3 bra $L__BB206_5;
+; SM70-NEXT: @%p3 bra $L__BB275_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end18
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd17+24];
-; SM70-NEXT: $L__BB206_7: // %atomicrmw.start24
+; SM70-NEXT: $L__BB275_7: // %atomicrmw.start24
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd10, %rd29;
; SM70-NEXT: min.f64 %rd21, %rd10, %rd4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
; SM70-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
-; SM70-NEXT: @%p4 bra $L__BB206_7;
+; SM70-NEXT: @%p4 bra $L__BB275_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end23
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd17+32];
-; SM70-NEXT: $L__BB206_9: // %atomicrmw.start38
+; SM70-NEXT: $L__BB275_9: // %atomicrmw.start38
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd13, %rd30;
; SM70-NEXT: min.f64 %rd22, %rd13, %rd1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
-; SM70-NEXT: @%p5 bra $L__BB206_9;
+; SM70-NEXT: @%p5 bra $L__BB275_9;
; SM70-NEXT: // %bb.10: // %atomicrmw.end37
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd17+40];
-; SM70-NEXT: $L__BB206_11: // %atomicrmw.start43
+; SM70-NEXT: $L__BB275_11: // %atomicrmw.start43
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd14, %rd31;
; SM70-NEXT: min.f64 %rd23, %rd14, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
; SM70-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
-; SM70-NEXT: @%p6 bra $L__BB206_11;
+; SM70-NEXT: @%p6 bra $L__BB275_11;
; SM70-NEXT: // %bb.12: // %atomicrmw.end42
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd17+48];
-; SM70-NEXT: $L__BB206_13: // %atomicrmw.start53
+; SM70-NEXT: $L__BB275_13: // %atomicrmw.start53
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: min.f64 %rd24, %rd32, %rd11;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
; SM70-NEXT: mov.b64 %rd32, %rd15;
-; SM70-NEXT: @%p7 bra $L__BB206_13;
+; SM70-NEXT: @%p7 bra $L__BB275_13;
; SM70-NEXT: // %bb.14: // %atomicrmw.end52
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd17+56];
-; SM70-NEXT: $L__BB206_15: // %atomicrmw.start58
+; SM70-NEXT: $L__BB275_15: // %atomicrmw.start58
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: min.f64 %rd25, %rd33, %rd12;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
; SM70-NEXT: mov.b64 %rd33, %rd16;
-; SM70-NEXT: @%p8 bra $L__BB206_15;
+; SM70-NEXT: @%p8 bra $L__BB275_15;
; SM70-NEXT: // %bb.16: // %atomicrmw.end57
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -9889,33 +11903,59 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x double> %retval
}
-define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
-; SM70-LABEL: fmax_acq_rel_v2f64_global_cta(
+define <1 x double> @fmax_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM70-LABEL: fmax_acq_rel_v1f64_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<3>;
-; SM70-NEXT: .reg .b64 %rd<10>;
-; SM70-EMPTY:
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM70-NEXT: $L__BB276_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd4, %rd5, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM70-NEXT: mov.b64 %rd5, %rd2;
+; SM70-NEXT: @%p1 bra $L__BB276_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM70-LABEL: fmax_acq_rel_v2f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd5, [fmax_acq_rel_v2f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd5];
-; SM70-NEXT: $L__BB207_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB277_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.f64 %rd6, %rd8, %rd1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
; SM70-NEXT: mov.b64 %rd8, %rd3;
-; SM70-NEXT: @%p1 bra $L__BB207_1;
+; SM70-NEXT: @%p1 bra $L__BB277_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd5+8];
-; SM70-NEXT: $L__BB207_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB277_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.f64 %rd7, %rd9, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
; SM70-NEXT: mov.b64 %rd9, %rd4;
-; SM70-NEXT: @%p2 bra $L__BB207_3;
+; SM70-NEXT: @%p2 bra $L__BB277_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -9936,40 +11976,40 @@ define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd9];
-; SM70-NEXT: $L__BB208_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB278_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd5, %rd14;
; SM70-NEXT: max.f64 %rd10, %rd5, %rd3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
; SM70-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
-; SM70-NEXT: @%p1 bra $L__BB208_1;
+; SM70-NEXT: @%p1 bra $L__BB278_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd9+8];
-; SM70-NEXT: $L__BB208_3: // %atomicrmw.start5
+; SM70-NEXT: $L__BB278_3: // %atomicrmw.start5
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd6, %rd15;
; SM70-NEXT: max.f64 %rd11, %rd6, %rd4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
; SM70-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
-; SM70-NEXT: @%p2 bra $L__BB208_3;
+; SM70-NEXT: @%p2 bra $L__BB278_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end4
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd9+16];
-; SM70-NEXT: $L__BB208_5: // %atomicrmw.start15
+; SM70-NEXT: $L__BB278_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.f64 %rd12, %rd16, %rd1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
; SM70-NEXT: mov.b64 %rd16, %rd7;
-; SM70-NEXT: @%p3 bra $L__BB208_5;
+; SM70-NEXT: @%p3 bra $L__BB278_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd9+24];
-; SM70-NEXT: $L__BB208_7: // %atomicrmw.start20
+; SM70-NEXT: $L__BB278_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.f64 %rd13, %rd17, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
; SM70-NEXT: mov.b64 %rd17, %rd8;
-; SM70-NEXT: @%p4 bra $L__BB208_7;
+; SM70-NEXT: @%p4 bra $L__BB278_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -9993,76 +12033,76 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmax_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd17];
-; SM70-NEXT: $L__BB209_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB279_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd7, %rd26;
; SM70-NEXT: max.f64 %rd18, %rd7, %rd5;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
; SM70-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
-; SM70-NEXT: @%p1 bra $L__BB209_1;
+; SM70-NEXT: @%p1 bra $L__BB279_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd17+8];
-; SM70-NEXT: $L__BB209_3: // %atomicrmw.start9
+; SM70-NEXT: $L__BB279_3: // %atomicrmw.start9
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd8, %rd27;
; SM70-NEXT: max.f64 %rd19, %rd8, %rd6;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
; SM70-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
-; SM70-NEXT: @%p2 bra $L__BB209_3;
+; SM70-NEXT: @%p2 bra $L__BB279_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end8
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd17+16];
-; SM70-NEXT: $L__BB209_5: // %atomicrmw.start19
+; SM70-NEXT: $L__BB279_5: // %atomicrmw.start19
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd9, %rd28;
; SM70-NEXT: max.f64 %rd20, %rd9, %rd3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
; SM70-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
-; SM70-NEXT: @%p3 bra $L__BB209_5;
+; SM70-NEXT: @%p3 bra $L__BB279_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end18
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd17+24];
-; SM70-NEXT: $L__BB209_7: // %atomicrmw.start24
+; SM70-NEXT: $L__BB279_7: // %atomicrmw.start24
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd10, %rd29;
; SM70-NEXT: max.f64 %rd21, %rd10, %rd4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
; SM70-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
-; SM70-NEXT: @%p4 bra $L__BB209_7;
+; SM70-NEXT: @%p4 bra $L__BB279_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end23
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd17+32];
-; SM70-NEXT: $L__BB209_9: // %atomicrmw.start38
+; SM70-NEXT: $L__BB279_9: // %atomicrmw.start38
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd13, %rd30;
; SM70-NEXT: max.f64 %rd22, %rd13, %rd1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
-; SM70-NEXT: @%p5 bra $L__BB209_9;
+; SM70-NEXT: @%p5 bra $L__BB279_9;
; SM70-NEXT: // %bb.10: // %atomicrmw.end37
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd17+40];
-; SM70-NEXT: $L__BB209_11: // %atomicrmw.start43
+; SM70-NEXT: $L__BB279_11: // %atomicrmw.start43
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd14, %rd31;
; SM70-NEXT: max.f64 %rd23, %rd14, %rd2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
; SM70-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
-; SM70-NEXT: @%p6 bra $L__BB209_11;
+; SM70-NEXT: @%p6 bra $L__BB279_11;
; SM70-NEXT: // %bb.12: // %atomicrmw.end42
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd17+48];
-; SM70-NEXT: $L__BB209_13: // %atomicrmw.start53
+; SM70-NEXT: $L__BB279_13: // %atomicrmw.start53
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.f64 %rd24, %rd32, %rd11;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
; SM70-NEXT: mov.b64 %rd32, %rd15;
-; SM70-NEXT: @%p7 bra $L__BB209_13;
+; SM70-NEXT: @%p7 bra $L__BB279_13;
; SM70-NEXT: // %bb.14: // %atomicrmw.end52
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd17+56];
-; SM70-NEXT: $L__BB209_15: // %atomicrmw.start58
+; SM70-NEXT: $L__BB279_15: // %atomicrmw.start58
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: max.f64 %rd25, %rd33, %rd12;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
; SM70-NEXT: mov.b64 %rd33, %rd16;
-; SM70-NEXT: @%p8 bra $L__BB209_15;
+; SM70-NEXT: @%p8 bra $L__BB279_15;
; SM70-NEXT: // %bb.16: // %atomicrmw.end57
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -10074,6 +12114,40 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x double> %retval
}
+define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM70-LABEL: fminimum_acq_rel_v1f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd3];
+; SM70-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM70-NEXT: $L__BB280_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
+; SM70-NEXT: min.f64 %rd4, %rd9, %rd1;
+; SM70-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd9, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
+; SM70-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
+; SM70-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
+; SM70-NEXT: mov.b64 %rd9, %rd2;
+; SM70-NEXT: @%p5 bra $L__BB280_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM70-LABEL: fminimum_acq_rel_v2f64_global_cta(
; SM70: {
@@ -10086,7 +12160,7 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd5];
; SM70-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
-; SM70-NEXT: $L__BB210_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB281_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
; SM70-NEXT: min.f64 %rd6, %rd16, %rd1;
@@ -10099,11 +12173,11 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
; SM70-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
; SM70-NEXT: mov.b64 %rd16, %rd3;
-; SM70-NEXT: @%p5 bra $L__BB210_1;
+; SM70-NEXT: @%p5 bra $L__BB281_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd5+8];
; SM70-NEXT: setp.eq.b64 %p8, %rd2, -9223372036854775808;
-; SM70-NEXT: $L__BB210_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB281_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
; SM70-NEXT: min.f64 %rd11, %rd17, %rd2;
@@ -10116,7 +12190,7 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
; SM70-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
; SM70-NEXT: mov.b64 %rd17, %rd4;
-; SM70-NEXT: @%p10 bra $L__BB210_3;
+; SM70-NEXT: @%p10 bra $L__BB281_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -10138,7 +12212,7 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd9];
; SM70-NEXT: setp.eq.b64 %p3, %rd3, -9223372036854775808;
-; SM70-NEXT: $L__BB211_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB282_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd5, %rd30;
; SM70-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
@@ -10151,11 +12225,11 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
-; SM70-NEXT: @%p5 bra $L__BB211_1;
+; SM70-NEXT: @%p5 bra $L__BB282_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd9+8];
; SM70-NEXT: setp.eq.b64 %p8, %rd4, -9223372036854775808;
-; SM70-NEXT: $L__BB211_3: // %atomicrmw.start5
+; SM70-NEXT: $L__BB282_3: // %atomicrmw.start5
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd6, %rd31;
; SM70-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
@@ -10168,11 +12242,11 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
; SM70-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
-; SM70-NEXT: @%p10 bra $L__BB211_3;
+; SM70-NEXT: @%p10 bra $L__BB282_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end4
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd9+16];
; SM70-NEXT: setp.eq.b64 %p13, %rd1, -9223372036854775808;
-; SM70-NEXT: $L__BB211_5: // %atomicrmw.start15
+; SM70-NEXT: $L__BB282_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
; SM70-NEXT: min.f64 %rd20, %rd32, %rd1;
@@ -10185,11 +12259,11 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
; SM70-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
; SM70-NEXT: mov.b64 %rd32, %rd7;
-; SM70-NEXT: @%p15 bra $L__BB211_5;
+; SM70-NEXT: @%p15 bra $L__BB282_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd9+24];
; SM70-NEXT: setp.eq.b64 %p18, %rd2, -9223372036854775808;
-; SM70-NEXT: $L__BB211_7: // %atomicrmw.start20
+; SM70-NEXT: $L__BB282_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
; SM70-NEXT: min.f64 %rd25, %rd33, %rd2;
@@ -10202,7 +12276,7 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
; SM70-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
; SM70-NEXT: mov.b64 %rd33, %rd8;
-; SM70-NEXT: @%p20 bra $L__BB211_7;
+; SM70-NEXT: @%p20 bra $L__BB282_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -10227,7 +12301,7 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd58, [%rd17];
; SM70-NEXT: setp.eq.b64 %p3, %rd5, -9223372036854775808;
-; SM70-NEXT: $L__BB212_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB283_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd7, %rd58;
; SM70-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
@@ -10240,11 +12314,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
; SM70-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
-; SM70-NEXT: @%p5 bra $L__BB212_1;
+; SM70-NEXT: @%p5 bra $L__BB283_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd59, [%rd17+8];
; SM70-NEXT: setp.eq.b64 %p8, %rd6, -9223372036854775808;
-; SM70-NEXT: $L__BB212_3: // %atomicrmw.start9
+; SM70-NEXT: $L__BB283_3: // %atomicrmw.start9
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd8, %rd59;
; SM70-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
@@ -10257,11 +12331,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
; SM70-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
-; SM70-NEXT: @%p10 bra $L__BB212_3;
+; SM70-NEXT: @%p10 bra $L__BB283_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end8
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd60, [%rd17+16];
; SM70-NEXT: setp.eq.b64 %p13, %rd3, -9223372036854775808;
-; SM70-NEXT: $L__BB212_5: // %atomicrmw.start19
+; SM70-NEXT: $L__BB283_5: // %atomicrmw.start19
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd9, %rd60;
; SM70-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
@@ -10274,11 +12348,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
; SM70-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
-; SM70-NEXT: @%p15 bra $L__BB212_5;
+; SM70-NEXT: @%p15 bra $L__BB283_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end18
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd61, [%rd17+24];
; SM70-NEXT: setp.eq.b64 %p18, %rd4, -9223372036854775808;
-; SM70-NEXT: $L__BB212_7: // %atomicrmw.start24
+; SM70-NEXT: $L__BB283_7: // %atomicrmw.start24
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd10, %rd61;
; SM70-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
@@ -10291,11 +12365,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
; SM70-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
-; SM70-NEXT: @%p20 bra $L__BB212_7;
+; SM70-NEXT: @%p20 bra $L__BB283_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end23
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd62, [%rd17+32];
; SM70-NEXT: setp.eq.b64 %p23, %rd1, -9223372036854775808;
-; SM70-NEXT: $L__BB212_9: // %atomicrmw.start38
+; SM70-NEXT: $L__BB283_9: // %atomicrmw.start38
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd13, %rd62;
; SM70-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
@@ -10308,11 +12382,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
; SM70-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
-; SM70-NEXT: @%p25 bra $L__BB212_9;
+; SM70-NEXT: @%p25 bra $L__BB283_9;
; SM70-NEXT: // %bb.10: // %atomicrmw.end37
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd63, [%rd17+40];
; SM70-NEXT: setp.eq.b64 %p28, %rd2, -9223372036854775808;
-; SM70-NEXT: $L__BB212_11: // %atomicrmw.start43
+; SM70-NEXT: $L__BB283_11: // %atomicrmw.start43
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd14, %rd63;
; SM70-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
@@ -10325,11 +12399,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
; SM70-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
-; SM70-NEXT: @%p30 bra $L__BB212_11;
+; SM70-NEXT: @%p30 bra $L__BB283_11;
; SM70-NEXT: // %bb.12: // %atomicrmw.end42
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd64, [%rd17+48];
; SM70-NEXT: setp.eq.b64 %p33, %rd11, -9223372036854775808;
-; SM70-NEXT: $L__BB212_13: // %atomicrmw.start53
+; SM70-NEXT: $L__BB283_13: // %atomicrmw.start53
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
; SM70-NEXT: min.f64 %rd48, %rd64, %rd11;
@@ -10342,11 +12416,11 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
; SM70-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
; SM70-NEXT: mov.b64 %rd64, %rd15;
-; SM70-NEXT: @%p35 bra $L__BB212_13;
+; SM70-NEXT: @%p35 bra $L__BB283_13;
; SM70-NEXT: // %bb.14: // %atomicrmw.end52
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd65, [%rd17+56];
; SM70-NEXT: setp.eq.b64 %p38, %rd12, -9223372036854775808;
-; SM70-NEXT: $L__BB212_15: // %atomicrmw.start58
+; SM70-NEXT: $L__BB283_15: // %atomicrmw.start58
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
; SM70-NEXT: min.f64 %rd53, %rd65, %rd12;
@@ -10359,7 +12433,7 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
; SM70-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
; SM70-NEXT: mov.b64 %rd65, %rd16;
-; SM70-NEXT: @%p40 bra $L__BB212_15;
+; SM70-NEXT: @%p40 bra $L__BB283_15;
; SM70-NEXT: // %bb.16: // %atomicrmw.end57
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -10371,6 +12445,40 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
ret <8 x double> %retval
}
+define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v1f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd3];
+; SM70-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM70-NEXT: $L__BB284_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
+; SM70-NEXT: max.f64 %rd4, %rd9, %rd1;
+; SM70-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd9, 0;
+; SM70-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
+; SM70-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
+; SM70-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
+; SM70-NEXT: mov.b64 %rd9, %rd2;
+; SM70-NEXT: @%p5 bra $L__BB284_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM70-LABEL: fmaximum_acq_rel_v2f64_global_cta(
; SM70: {
@@ -10383,7 +12491,7 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd5];
; SM70-NEXT: setp.eq.b64 %p3, %rd1, 0;
-; SM70-NEXT: $L__BB213_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB285_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
; SM70-NEXT: max.f64 %rd6, %rd16, %rd1;
@@ -10396,11 +12504,11 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
; SM70-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
; SM70-NEXT: mov.b64 %rd16, %rd3;
-; SM70-NEXT: @%p5 bra $L__BB213_1;
+; SM70-NEXT: @%p5 bra $L__BB285_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd5+8];
; SM70-NEXT: setp.eq.b64 %p8, %rd2, 0;
-; SM70-NEXT: $L__BB213_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB285_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
; SM70-NEXT: max.f64 %rd11, %rd17, %rd2;
@@ -10413,7 +12521,7 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
; SM70-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
; SM70-NEXT: mov.b64 %rd17, %rd4;
-; SM70-NEXT: @%p10 bra $L__BB213_3;
+; SM70-NEXT: @%p10 bra $L__BB285_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -10435,7 +12543,7 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd9];
; SM70-NEXT: setp.eq.b64 %p3, %rd3, 0;
-; SM70-NEXT: $L__BB214_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB286_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd5, %rd30;
; SM70-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
@@ -10448,11 +12556,11 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
-; SM70-NEXT: @%p5 bra $L__BB214_1;
+; SM70-NEXT: @%p5 bra $L__BB286_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd9+8];
; SM70-NEXT: setp.eq.b64 %p8, %rd4, 0;
-; SM70-NEXT: $L__BB214_3: // %atomicrmw.start5
+; SM70-NEXT: $L__BB286_3: // %atomicrmw.start5
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd6, %rd31;
; SM70-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
@@ -10465,11 +12573,11 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
; SM70-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
-; SM70-NEXT: @%p10 bra $L__BB214_3;
+; SM70-NEXT: @%p10 bra $L__BB286_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end4
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd9+16];
; SM70-NEXT: setp.eq.b64 %p13, %rd1, 0;
-; SM70-NEXT: $L__BB214_5: // %atomicrmw.start15
+; SM70-NEXT: $L__BB286_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
; SM70-NEXT: max.f64 %rd20, %rd32, %rd1;
@@ -10482,11 +12590,11 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
; SM70-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
; SM70-NEXT: mov.b64 %rd32, %rd7;
-; SM70-NEXT: @%p15 bra $L__BB214_5;
+; SM70-NEXT: @%p15 bra $L__BB286_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd9+24];
; SM70-NEXT: setp.eq.b64 %p18, %rd2, 0;
-; SM70-NEXT: $L__BB214_7: // %atomicrmw.start20
+; SM70-NEXT: $L__BB286_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
; SM70-NEXT: max.f64 %rd25, %rd33, %rd2;
@@ -10499,7 +12607,7 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
; SM70-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
; SM70-NEXT: mov.b64 %rd33, %rd8;
-; SM70-NEXT: @%p20 bra $L__BB214_7;
+; SM70-NEXT: @%p20 bra $L__BB286_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -10524,7 +12632,7 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd58, [%rd17];
; SM70-NEXT: setp.eq.b64 %p3, %rd5, 0;
-; SM70-NEXT: $L__BB215_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB287_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd7, %rd58;
; SM70-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
@@ -10537,11 +12645,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
; SM70-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
-; SM70-NEXT: @%p5 bra $L__BB215_1;
+; SM70-NEXT: @%p5 bra $L__BB287_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd59, [%rd17+8];
; SM70-NEXT: setp.eq.b64 %p8, %rd6, 0;
-; SM70-NEXT: $L__BB215_3: // %atomicrmw.start9
+; SM70-NEXT: $L__BB287_3: // %atomicrmw.start9
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd8, %rd59;
; SM70-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
@@ -10554,11 +12662,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
; SM70-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
-; SM70-NEXT: @%p10 bra $L__BB215_3;
+; SM70-NEXT: @%p10 bra $L__BB287_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end8
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd60, [%rd17+16];
; SM70-NEXT: setp.eq.b64 %p13, %rd3, 0;
-; SM70-NEXT: $L__BB215_5: // %atomicrmw.start19
+; SM70-NEXT: $L__BB287_5: // %atomicrmw.start19
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd9, %rd60;
; SM70-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
@@ -10571,11 +12679,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
; SM70-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
-; SM70-NEXT: @%p15 bra $L__BB215_5;
+; SM70-NEXT: @%p15 bra $L__BB287_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end18
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd61, [%rd17+24];
; SM70-NEXT: setp.eq.b64 %p18, %rd4, 0;
-; SM70-NEXT: $L__BB215_7: // %atomicrmw.start24
+; SM70-NEXT: $L__BB287_7: // %atomicrmw.start24
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd10, %rd61;
; SM70-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
@@ -10588,11 +12696,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
; SM70-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
-; SM70-NEXT: @%p20 bra $L__BB215_7;
+; SM70-NEXT: @%p20 bra $L__BB287_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end23
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd62, [%rd17+32];
; SM70-NEXT: setp.eq.b64 %p23, %rd1, 0;
-; SM70-NEXT: $L__BB215_9: // %atomicrmw.start38
+; SM70-NEXT: $L__BB287_9: // %atomicrmw.start38
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd13, %rd62;
; SM70-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
@@ -10605,11 +12713,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
; SM70-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
-; SM70-NEXT: @%p25 bra $L__BB215_9;
+; SM70-NEXT: @%p25 bra $L__BB287_9;
; SM70-NEXT: // %bb.10: // %atomicrmw.end37
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd63, [%rd17+40];
; SM70-NEXT: setp.eq.b64 %p28, %rd2, 0;
-; SM70-NEXT: $L__BB215_11: // %atomicrmw.start43
+; SM70-NEXT: $L__BB287_11: // %atomicrmw.start43
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b64 %rd14, %rd63;
; SM70-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
@@ -10622,11 +12730,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
; SM70-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
-; SM70-NEXT: @%p30 bra $L__BB215_11;
+; SM70-NEXT: @%p30 bra $L__BB287_11;
; SM70-NEXT: // %bb.12: // %atomicrmw.end42
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd64, [%rd17+48];
; SM70-NEXT: setp.eq.b64 %p33, %rd11, 0;
-; SM70-NEXT: $L__BB215_13: // %atomicrmw.start53
+; SM70-NEXT: $L__BB287_13: // %atomicrmw.start53
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
; SM70-NEXT: max.f64 %rd48, %rd64, %rd11;
@@ -10639,11 +12747,11 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
; SM70-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
; SM70-NEXT: mov.b64 %rd64, %rd15;
-; SM70-NEXT: @%p35 bra $L__BB215_13;
+; SM70-NEXT: @%p35 bra $L__BB287_13;
; SM70-NEXT: // %bb.14: // %atomicrmw.end52
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd65, [%rd17+56];
; SM70-NEXT: setp.eq.b64 %p38, %rd12, 0;
-; SM70-NEXT: $L__BB215_15: // %atomicrmw.start58
+; SM70-NEXT: $L__BB287_15: // %atomicrmw.start58
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
; SM70-NEXT: max.f64 %rd53, %rd65, %rd12;
@@ -10656,7 +12764,7 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
; SM70-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
; SM70-NEXT: mov.b64 %rd65, %rd16;
-; SM70-NEXT: @%p40 bra $L__BB215_15;
+; SM70-NEXT: @%p40 bra $L__BB287_15;
; SM70-NEXT: // %bb.16: // %atomicrmw.end57
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -10668,6 +12776,24 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
ret <8 x double> %retval
}
+define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM70-LABEL: fadd_acq_rel_v1f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b16 %rs<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-LABEL: fadd_acq_rel_v2f16_global_cta(
; SM70: {
@@ -10742,6 +12868,48 @@ define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
ret <8 x half> %retval
}
+define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM70-LABEL: fsub_acq_rel_v1f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB292_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB292_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM70: {
@@ -10754,13 +12922,13 @@ define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NEXT: $L__BB219_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB293_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f16x2 %r3, %r4, %r2;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB219_1;
+; SM70-NEXT: @%p1 bra $L__BB293_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -10783,7 +12951,7 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB220_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB294_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f16x2 %r3, %r5, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -10799,7 +12967,7 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB220_1;
+; SM70-NEXT: @%p1 bra $L__BB294_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -10822,7 +12990,7 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB221_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB295_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f16x2 %r5, %r9, %r1;
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
@@ -10838,12 +13006,12 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB221_1;
+; SM70-NEXT: @%p1 bra $L__BB295_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM70-NEXT: $L__BB221_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB295_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: sub.rn.f16x2 %r7, %r11, %r3;
; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
@@ -10859,7 +13027,7 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB221_3;
+; SM70-NEXT: @%p2 bra $L__BB295_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -10868,6 +13036,51 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
ret <8 x half> %retval
}
+define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM70-LABEL: fmin_acq_rel_v1f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<18>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM70-NEXT: $L__BB296_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM70-NEXT: min.f32 %r11, %r9, %r10;
+; SM70-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NEXT: mov.b32 %r17, %r3;
+; SM70-NEXT: @%p1 bra $L__BB296_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-LABEL: fmin_acq_rel_v2f16_global_cta(
; SM70: {
@@ -10884,7 +13097,7 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT: cvt.f32.f16 %r3, %rs2;
; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
-; SM70-NEXT: $L__BB222_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB297_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r10;
; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
@@ -10897,7 +13110,7 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r10;
; SM70-NEXT: mov.b32 %r10, %r1;
-; SM70-NEXT: @%p1 bra $L__BB222_1;
+; SM70-NEXT: @%p1 bra $L__BB297_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -10927,7 +13140,7 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT: cvt.f32.f16 %r10, %rs8;
; SM70-NEXT: cvt.f32.f16 %r13, %rs7;
-; SM70-NEXT: $L__BB223_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB298_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r17;
; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
@@ -10957,7 +13170,7 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB223_1;
+; SM70-NEXT: @%p1 bra $L__BB298_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r17, %r18};
@@ -10985,7 +13198,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: cvt.f32.f16 %r5, %rs2;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT: cvt.f32.f16 %r12, %rs8;
-; SM70-NEXT: $L__BB224_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB299_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r33;
; SM70-NEXT: cvt.f32.f16 %r6, %rs4;
@@ -11017,7 +13230,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB224_1;
+; SM70-NEXT: @%p1 bra $L__BB299_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
@@ -11026,7 +13239,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: cvt.f32.f16 %r19, %rs14;
; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
; SM70-NEXT: cvt.f32.f16 %r26, %rs20;
-; SM70-NEXT: $L__BB224_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB299_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r35;
; SM70-NEXT: cvt.f32.f16 %r20, %rs16;
@@ -11058,7 +13271,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r35, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB224_3;
+; SM70-NEXT: @%p2 bra $L__BB299_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
@@ -11067,6 +13280,51 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
ret <8 x half> %retval
}
+define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM70-LABEL: fmax_acq_rel_v1f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<18>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM70-NEXT: $L__BB300_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM70-NEXT: max.f32 %r11, %r9, %r10;
+; SM70-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NEXT: mov.b32 %r17, %r3;
+; SM70-NEXT: @%p1 bra $L__BB300_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-LABEL: fmax_acq_rel_v2f16_global_cta(
; SM70: {
@@ -11083,7 +13341,7 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT: cvt.f32.f16 %r3, %rs2;
; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
-; SM70-NEXT: $L__BB225_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB301_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r10;
; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
@@ -11096,7 +13354,7 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r10;
; SM70-NEXT: mov.b32 %r10, %r1;
-; SM70-NEXT: @%p1 bra $L__BB225_1;
+; SM70-NEXT: @%p1 bra $L__BB301_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -11126,7 +13384,7 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT: cvt.f32.f16 %r10, %rs8;
; SM70-NEXT: cvt.f32.f16 %r13, %rs7;
-; SM70-NEXT: $L__BB226_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB302_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r17;
; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
@@ -11156,7 +13414,7 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB226_1;
+; SM70-NEXT: @%p1 bra $L__BB302_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r17, %r18};
@@ -11184,7 +13442,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: cvt.f32.f16 %r5, %rs2;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT: cvt.f32.f16 %r12, %rs8;
-; SM70-NEXT: $L__BB227_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB303_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r33;
; SM70-NEXT: cvt.f32.f16 %r6, %rs4;
@@ -11216,7 +13474,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB227_1;
+; SM70-NEXT: @%p1 bra $L__BB303_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
@@ -11225,7 +13483,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: cvt.f32.f16 %r19, %rs14;
; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
; SM70-NEXT: cvt.f32.f16 %r26, %rs20;
-; SM70-NEXT: $L__BB227_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB303_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r35;
; SM70-NEXT: cvt.f32.f16 %r20, %rs16;
@@ -11257,7 +13515,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r35, %rd21;
-; SM70-NEXT: @%p2 bra $L__BB227_3;
+; SM70-NEXT: @%p2 bra $L__BB303_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
@@ -11266,6 +13524,58 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
ret <8 x half> %retval
}
+define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM70-LABEL: fminimum_acq_rel_v1f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<7>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM70-NEXT: $L__BB304_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: setp.lt.f16 %p1, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-NEXT: mov.b16 %rs7, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
+; SM70-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p6 bra $L__BB304_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-LABEL: fminimum_acq_rel_v2f16_global_cta(
; SM70: {
@@ -11282,7 +13592,7 @@ define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
-; SM70-NEXT: $L__BB228_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB305_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
@@ -11308,7 +13618,7 @@ define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p11, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p11 bra $L__BB228_1;
+; SM70-NEXT: @%p11 bra $L__BB305_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -11338,7 +13648,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
; SM70-NEXT: setp.eq.b16 %p14, %rs17, -32768;
; SM70-NEXT: setp.eq.b16 %p19, %rs16, -32768;
-; SM70-NEXT: $L__BB229_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB306_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
; SM70-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
@@ -11393,7 +13703,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB229_1;
+; SM70-NEXT: @%p21 bra $L__BB306_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -11423,7 +13733,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
; SM70-NEXT: setp.eq.b16 %p14, %rs17, -32768;
; SM70-NEXT: setp.eq.b16 %p19, %rs16, -32768;
-; SM70-NEXT: $L__BB230_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB307_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
; SM70-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
@@ -11478,12 +13788,12 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB230_1;
+; SM70-NEXT: @%p21 bra $L__BB307_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM70-NEXT: $L__BB230_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB307_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs30, %rs31}, %r3;
; SM70-NEXT: mov.b32 {%rs32, %rs33}, %r11;
@@ -11544,7 +13854,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p42 bra $L__BB230_3;
+; SM70-NEXT: @%p42 bra $L__BB307_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -11553,6 +13863,58 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x half> %retval
}
+define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v1f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<7>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM70-NEXT: $L__BB308_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: setp.gt.f16 %p1, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-NEXT: mov.b16 %rs7, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
+; SM70-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p6 bra $L__BB308_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-LABEL: fmaximum_acq_rel_v2f16_global_cta(
; SM70: {
@@ -11569,7 +13931,7 @@ define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
-; SM70-NEXT: $L__BB231_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB309_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
@@ -11595,7 +13957,7 @@ define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT: setp.ne.b32 %p11, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p11 bra $L__BB231_1;
+; SM70-NEXT: @%p11 bra $L__BB309_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -11625,7 +13987,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
; SM70-NEXT: setp.eq.b16 %p14, %rs17, 0;
; SM70-NEXT: setp.eq.b16 %p19, %rs16, 0;
-; SM70-NEXT: $L__BB232_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB310_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
; SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
@@ -11680,7 +14042,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB232_1;
+; SM70-NEXT: @%p21 bra $L__BB310_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -11710,7 +14072,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
; SM70-NEXT: setp.eq.b16 %p14, %rs17, 0;
; SM70-NEXT: setp.eq.b16 %p19, %rs16, 0;
-; SM70-NEXT: $L__BB233_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB311_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
; SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
@@ -11765,12 +14127,12 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB233_1;
+; SM70-NEXT: @%p21 bra $L__BB311_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
-; SM70-NEXT: $L__BB233_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB311_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs30, %rs31}, %r3;
; SM70-NEXT: mov.b32 {%rs32, %rs33}, %r11;
@@ -11831,7 +14193,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p42 bra $L__BB233_3;
+; SM70-NEXT: @%p42 bra $L__BB311_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -11840,6 +14202,56 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x half> %retval
}
+define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM70-LABEL: fadd_acq_rel_v1bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<2>;
+; SM70-NEXT: .reg .b32 %r<24>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NEXT: $L__BB312_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NEXT: mov.b32 %r23, %r3;
+; SM70-NEXT: @%p2 bra $L__BB312_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM70-LABEL: fadd_acq_rel_v2bf16_global_cta(
; SM70: {
@@ -11858,7 +14270,7 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: shl.b32 %r4, %r3, 16;
; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: $L__BB234_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB313_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r24;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -11883,7 +14295,7 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
; SM70-NEXT: setp.ne.b32 %p3, %r1, %r24;
; SM70-NEXT: mov.b32 %r24, %r1;
-; SM70-NEXT: @%p3 bra $L__BB234_1;
+; SM70-NEXT: @%p3 bra $L__BB313_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -11917,7 +14329,7 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: shl.b32 %r25, %r24, 16;
; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
; SM70-NEXT: shl.b32 %r35, %r34, 16;
-; SM70-NEXT: $L__BB235_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB314_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -11971,7 +14383,7 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r45, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB235_1;
+; SM70-NEXT: @%p5 bra $L__BB314_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
@@ -12001,7 +14413,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
; SM70-NEXT: shl.b32 %r27, %r26, 16;
-; SM70-NEXT: $L__BB236_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB315_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
@@ -12059,14 +14471,14 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r89, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB236_1;
+; SM70-NEXT: @%p5 bra $L__BB315_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r91, %rd12;
; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r4;
-; SM70-NEXT: $L__BB236_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB315_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u32.u16 %r47, %rs10;
; SM70-NEXT: shl.b32 %r48, %r47, 16;
@@ -12128,7 +14540,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r91, %rd21;
-; SM70-NEXT: @%p10 bra $L__BB236_3;
+; SM70-NEXT: @%p10 bra $L__BB315_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
@@ -12137,6 +14549,56 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x bfloat> %retval
}
+define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM70-LABEL: fsub_acq_rel_v1bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<2>;
+; SM70-NEXT: .reg .b32 %r<24>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NEXT: $L__BB316_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: sub.rn.f32 %r12, %r9, %r11;
+; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NEXT: mov.b32 %r23, %r3;
+; SM70-NEXT: @%p2 bra $L__BB316_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM70-LABEL: fsub_acq_rel_v2bf16_global_cta(
; SM70: {
@@ -12155,7 +14617,7 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: shl.b32 %r4, %r3, 16;
; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: $L__BB237_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB317_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r24;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -12180,7 +14642,7 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
; SM70-NEXT: setp.ne.b32 %p3, %r1, %r24;
; SM70-NEXT: mov.b32 %r24, %r1;
-; SM70-NEXT: @%p3 bra $L__BB237_1;
+; SM70-NEXT: @%p3 bra $L__BB317_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -12214,7 +14676,7 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: shl.b32 %r25, %r24, 16;
; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
; SM70-NEXT: shl.b32 %r35, %r34, 16;
-; SM70-NEXT: $L__BB238_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB318_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -12268,7 +14730,7 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r45, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB238_1;
+; SM70-NEXT: @%p5 bra $L__BB318_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
@@ -12298,7 +14760,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
; SM70-NEXT: shl.b32 %r27, %r26, 16;
-; SM70-NEXT: $L__BB239_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB319_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
@@ -12356,14 +14818,14 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r89, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB239_1;
+; SM70-NEXT: @%p5 bra $L__BB319_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r91, %rd12;
; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r4;
-; SM70-NEXT: $L__BB239_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB319_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u32.u16 %r47, %rs10;
; SM70-NEXT: shl.b32 %r48, %r47, 16;
@@ -12425,7 +14887,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r91, %rd21;
-; SM70-NEXT: @%p10 bra $L__BB239_3;
+; SM70-NEXT: @%p10 bra $L__BB319_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
@@ -12434,6 +14896,56 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x bfloat> %retval
}
+define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM70-LABEL: fmin_acq_rel_v1bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<2>;
+; SM70-NEXT: .reg .b32 %r<24>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NEXT: $L__BB320_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: min.f32 %r12, %r9, %r11;
+; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NEXT: mov.b32 %r23, %r3;
+; SM70-NEXT: @%p2 bra $L__BB320_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM70-LABEL: fmin_acq_rel_v2bf16_global_cta(
; SM70: {
@@ -12452,7 +14964,7 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: shl.b32 %r4, %r3, 16;
; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: $L__BB240_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB321_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r24;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -12477,7 +14989,7 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
; SM70-NEXT: setp.ne.b32 %p3, %r1, %r24;
; SM70-NEXT: mov.b32 %r24, %r1;
-; SM70-NEXT: @%p3 bra $L__BB240_1;
+; SM70-NEXT: @%p3 bra $L__BB321_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -12511,7 +15023,7 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: shl.b32 %r25, %r24, 16;
; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
; SM70-NEXT: shl.b32 %r35, %r34, 16;
-; SM70-NEXT: $L__BB241_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB322_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -12565,7 +15077,7 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r45, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB241_1;
+; SM70-NEXT: @%p5 bra $L__BB322_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
@@ -12595,7 +15107,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
; SM70-NEXT: shl.b32 %r27, %r26, 16;
-; SM70-NEXT: $L__BB242_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB323_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
@@ -12653,14 +15165,14 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r89, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB242_1;
+; SM70-NEXT: @%p5 bra $L__BB323_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r91, %rd12;
; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r4;
-; SM70-NEXT: $L__BB242_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB323_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u32.u16 %r47, %rs10;
; SM70-NEXT: shl.b32 %r48, %r47, 16;
@@ -12722,7 +15234,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r91, %rd21;
-; SM70-NEXT: @%p10 bra $L__BB242_3;
+; SM70-NEXT: @%p10 bra $L__BB323_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
@@ -12731,13 +15243,63 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x bfloat> %retval
}
-define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
-; SM70-LABEL: fmax_acq_rel_v2bf16_global_cta(
+define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM70-LABEL: fmax_acq_rel_v1bf16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<4>;
-; SM70-NEXT: .reg .b16 %rs<5>;
-; SM70-NEXT: .reg .b32 %r<25>;
-; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<2>;
+; SM70-NEXT: .reg .b32 %r<24>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NEXT: $L__BB324_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: max.f32 %r12, %r9, %r11;
+; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NEXT: mov.b32 %r23, %r3;
+; SM70-NEXT: @%p2 bra $L__BB324_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM70-LABEL: fmax_acq_rel_v2bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
@@ -12749,7 +15311,7 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: shl.b32 %r4, %r3, 16;
; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: $L__BB243_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB325_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r24;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -12774,7 +15336,7 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
; SM70-NEXT: setp.ne.b32 %p3, %r1, %r24;
; SM70-NEXT: mov.b32 %r24, %r1;
-; SM70-NEXT: @%p3 bra $L__BB243_1;
+; SM70-NEXT: @%p3 bra $L__BB325_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -12808,7 +15370,7 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: shl.b32 %r25, %r24, 16;
; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
; SM70-NEXT: shl.b32 %r35, %r34, 16;
-; SM70-NEXT: $L__BB244_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB326_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -12862,7 +15424,7 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r45, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB244_1;
+; SM70-NEXT: @%p5 bra $L__BB326_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
@@ -12892,7 +15454,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
; SM70-NEXT: shl.b32 %r27, %r26, 16;
-; SM70-NEXT: $L__BB245_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB327_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
@@ -12950,14 +15512,14 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r89, %rd11;
-; SM70-NEXT: @%p5 bra $L__BB245_1;
+; SM70-NEXT: @%p5 bra $L__BB327_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r91, %rd12;
; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r3;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r4;
-; SM70-NEXT: $L__BB245_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB327_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u32.u16 %r47, %rs10;
; SM70-NEXT: shl.b32 %r48, %r47, 16;
@@ -13019,7 +15581,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r91, %rd21;
-; SM70-NEXT: @%p10 bra $L__BB245_3;
+; SM70-NEXT: @%p10 bra $L__BB327_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
@@ -13028,6 +15590,62 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x bfloat> %retval
}
+define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM70-LABEL: fminimum_acq_rel_v1bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<7>;
+; SM70-NEXT: .reg .b16 %rs<8>;
+; SM70-NEXT: .reg .b32 %r<20>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM70-NEXT: $L__BB328_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r19, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: setp.lt.f32 %p1, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM70-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
+; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM70-NEXT: shl.b32 %r15, %r14, %r1;
+; SM70-NEXT: and.b32 %r16, %r19, %r2;
+; SM70-NEXT: or.b32 %r17, %r16, %r15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM70-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM70-NEXT: mov.b32 %r19, %r3;
+; SM70-NEXT: @%p6 bra $L__BB328_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r18, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r18;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM70-LABEL: fminimum_acq_rel_v2bf16_global_cta(
; SM70: {
@@ -13048,7 +15666,7 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
; SM70-NEXT: shl.b32 %r10, %r9, 16;
; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
-; SM70-NEXT: $L__BB246_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB329_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r16;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -13081,7 +15699,7 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
; SM70-NEXT: setp.ne.b32 %p11, %r1, %r16;
; SM70-NEXT: mov.b32 %r16, %r1;
-; SM70-NEXT: @%p11 bra $L__BB246_1;
+; SM70-NEXT: @%p11 bra $L__BB329_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -13119,7 +15737,7 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: cvt.u32.u16 %r22, %rs15;
; SM70-NEXT: shl.b32 %r23, %r22, 16;
; SM70-NEXT: setp.eq.b16 %p19, %rs15, -32768;
-; SM70-NEXT: $L__BB247_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB330_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r29;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -13189,7 +15807,7 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r29, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB247_1;
+; SM70-NEXT: @%p21 bra $L__BB330_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
@@ -13223,7 +15841,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: shl.b32 %r19, %r18, 16;
; SM70-NEXT: setp.eq.b16 %p14, %rs16, -32768;
; SM70-NEXT: setp.eq.b16 %p19, %rs15, -32768;
-; SM70-NEXT: $L__BB248_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB331_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r57;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
@@ -13297,12 +15915,12 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r57, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB248_1;
+; SM70-NEXT: @%p21 bra $L__BB331_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r59, %rd12;
-; SM70-NEXT: $L__BB248_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB331_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs29, %rs30}, %r3;
; SM70-NEXT: cvt.u32.u16 %r31, %rs30;
@@ -13386,7 +16004,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r59, %rd21;
-; SM70-NEXT: @%p42 bra $L__BB248_3;
+; SM70-NEXT: @%p42 bra $L__BB331_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
@@ -13395,6 +16013,62 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
ret <8 x bfloat> %retval
}
+define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v1bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<7>;
+; SM70-NEXT: .reg .b16 %rs<8>;
+; SM70-NEXT: .reg .b32 %r<20>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM70-NEXT: $L__BB332_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r19, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: setp.gt.f32 %p1, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM70-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
+; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM70-NEXT: shl.b32 %r15, %r14, %r1;
+; SM70-NEXT: and.b32 %r16, %r19, %r2;
+; SM70-NEXT: or.b32 %r17, %r16, %r15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM70-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM70-NEXT: mov.b32 %r19, %r3;
+; SM70-NEXT: @%p6 bra $L__BB332_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r18, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r18;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM70-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
; SM70: {
@@ -13415,7 +16089,7 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
; SM70-NEXT: shl.b32 %r10, %r9, 16;
; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
-; SM70-NEXT: $L__BB249_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB333_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r16;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -13448,7 +16122,7 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
; SM70-NEXT: setp.ne.b32 %p11, %r1, %r16;
; SM70-NEXT: mov.b32 %r16, %r1;
-; SM70-NEXT: @%p11 bra $L__BB249_1;
+; SM70-NEXT: @%p11 bra $L__BB333_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -13486,7 +16160,7 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: cvt.u32.u16 %r22, %rs15;
; SM70-NEXT: shl.b32 %r23, %r22, 16;
; SM70-NEXT: setp.eq.b16 %p19, %rs15, 0;
-; SM70-NEXT: $L__BB250_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB334_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r29;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
@@ -13556,7 +16230,7 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r29, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB250_1;
+; SM70-NEXT: @%p21 bra $L__BB334_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
@@ -13590,7 +16264,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: shl.b32 %r19, %r18, 16;
; SM70-NEXT: setp.eq.b16 %p14, %rs16, 0;
; SM70-NEXT: setp.eq.b16 %p19, %rs15, 0;
-; SM70-NEXT: $L__BB251_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB335_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r57;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
@@ -13664,12 +16338,12 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r57, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB251_1;
+; SM70-NEXT: @%p21 bra $L__BB335_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r59, %rd12;
-; SM70-NEXT: $L__BB251_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB335_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs29, %rs30}, %r3;
; SM70-NEXT: cvt.u32.u16 %r31, %rs30;
@@ -13753,7 +16427,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r59, %rd21;
-; SM70-NEXT: @%p42 bra $L__BB251_3;
+; SM70-NEXT: @%p42 bra $L__BB335_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
@@ -13762,151 +16436,45 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
ret <8 x bfloat> %retval
}
-define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM70-LABEL: add_monotonic_v2i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<12>;
-; SM70-NEXT: .reg .b32 %r<16>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_monotonic_v2i8_global_cta_param_1];
-; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM70-NEXT: ld.param.b64 %rd2, [add_monotonic_v2i8_global_cta_param_0];
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: and.b32 %r6, %r5, 3;
-; SM70-NEXT: shl.b32 %r1, %r6, 3;
-; SM70-NEXT: mov.b32 %r7, 65535;
-; SM70-NEXT: shl.b32 %r8, %r7, %r1;
-; SM70-NEXT: not.b32 %r2, %r8;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
-; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM70-NEXT: $L__BB252_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r9, %r15, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
-; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
-; SM70-NEXT: and.b16 %rs9, %rs8, 255;
-; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
-; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
-; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
-; SM70-NEXT: shl.b32 %r11, %r10, %r1;
-; SM70-NEXT: and.b32 %r12, %r15, %r2;
-; SM70-NEXT: or.b32 %r13, %r12, %r11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
-; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
-; SM70-NEXT: mov.b32 %r15, %r3;
-; SM70-NEXT: @%p1 bra $L__BB252_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r14, %r3, %r1;
-; SM70-NEXT: st.param.b16 [func_retval0], %r14;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
- ret <2 x i8> %retval
-}
-
-define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM70-LABEL: add_acquire_v2i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<12>;
-; SM70-NEXT: .reg .b32 %r<16>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
-; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM70-NEXT: ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: and.b32 %r6, %r5, 3;
-; SM70-NEXT: shl.b32 %r1, %r6, 3;
-; SM70-NEXT: mov.b32 %r7, 65535;
-; SM70-NEXT: shl.b32 %r8, %r7, %r1;
-; SM70-NEXT: not.b32 %r2, %r8;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
-; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM70-NEXT: $L__BB253_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r9, %r15, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
-; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
-; SM70-NEXT: and.b16 %rs9, %rs8, 255;
-; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
-; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
-; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
-; SM70-NEXT: shl.b32 %r11, %r10, %r1;
-; SM70-NEXT: and.b32 %r12, %r15, %r2;
-; SM70-NEXT: or.b32 %r13, %r12, %r11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
-; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
-; SM70-NEXT: mov.b32 %r15, %r3;
-; SM70-NEXT: @%p1 bra $L__BB253_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r14, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r14;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
- ret <2 x i8> %retval
-}
-
-define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM70-LABEL: add_release_v2i8_global_cta(
+define <1 x i8> @add_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: add_monotonic_v1i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<12>;
; SM70-NEXT: .reg .b32 %r<16>;
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
-; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM70-NEXT: ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [add_monotonic_v1i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b8 %r6, [add_monotonic_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: and.b32 %r6, %r5, 3;
-; SM70-NEXT: shl.b32 %r1, %r6, 3;
-; SM70-NEXT: mov.b32 %r7, 65535;
-; SM70-NEXT: shl.b32 %r8, %r7, %r1;
-; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
-; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM70-NEXT: $L__BB254_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB336_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r9, %r15, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
-; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
-; SM70-NEXT: and.b16 %rs9, %rs8, 255;
-; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
-; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
-; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
-; SM70-NEXT: shl.b32 %r11, %r10, %r1;
-; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: add.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
; SM70-NEXT: or.b32 %r13, %r12, %r11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
-; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
-; SM70-NEXT: mov.b32 %r15, %r3;
-; SM70-NEXT: @%p1 bra $L__BB254_1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB336_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r14, %r3, %r1;
-; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
- ret <2 x i8> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
+ ret <1 x i8> %retval
}
-define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM70-LABEL: add_seq_cst_v2i8_global_cta(
+define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_monotonic_v2i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
; SM70-NEXT: .reg .b16 %rs<12>;
@@ -13914,10 +16482,9 @@ define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_monotonic_v2i8_global_cta_param_1];
; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM70-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [add_monotonic_v2i8_global_cta_param_0];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: and.b32 %r6, %r5, 3;
@@ -13927,7 +16494,7 @@ define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM70-NEXT: $L__BB255_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB337_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r15, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -13944,13 +16511,12 @@ define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
; SM70-NEXT: mov.b32 %r15, %r3;
-; SM70-NEXT: @%p1 bra $L__BB255_1;
+; SM70-NEXT: @%p1 bra $L__BB337_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r14, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b16 [func_retval0], %r14;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
ret <2 x i8> %retval
}
@@ -13974,7 +16540,7 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: $L__BB256_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB338_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -13998,7 +16564,7 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM70-NEXT: mov.b32 %r18, %r1;
-; SM70-NEXT: @%p1 bra $L__BB256_1;
+; SM70-NEXT: @%p1 bra $L__BB338_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NEXT: ret;
@@ -14006,114 +16572,184 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
ret <4 x i8> %retval
}
-define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM70-LABEL: add_acquire_v4i8_global_cta(
+define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: add_monotonic_v8i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<13>;
-; SM70-NEXT: .reg .b32 %r<19>;
-; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: $L__BB257_1: // %atomicrmw.start
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB339_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
-; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
-; SM70-NEXT: mov.b32 %r18, %r1;
-; SM70-NEXT: @%p1 bra $L__BB257_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
- ret <4 x i8> %retval
-}
-
-define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM70-LABEL: add_release_v4i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<13>;
-; SM70-NEXT: .reg .b32 %r<19>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: $L__BB258_1: // %atomicrmw.start
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM70-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM70-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM70-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB339_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @add_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: add_acquire_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [add_acquire_v1i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b8 %r6, [add_acquire_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB340_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
-; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
-; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
-; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
-; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
-; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
-; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
-; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
-; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
-; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
-; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
-; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
-; SM70-NEXT: mov.b32 %r18, %r1;
-; SM70-NEXT: @%p1 bra $L__BB258_1;
+; SM70-NEXT: add.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB340_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
- ret <4 x i8> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
+ ret <1 x i8> %retval
}
-define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM70-LABEL: add_seq_cst_v4i8_global_cta(
+define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_acquire_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB341_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB341_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: add_acquire_v4i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
; SM70-NEXT: .reg .b16 %rs<13>;
@@ -14121,9 +16757,8 @@ define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
@@ -14133,7 +16768,7 @@ define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: $L__BB259_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB342_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -14157,17 +16792,17 @@ define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM70-NEXT: mov.b32 %r18, %r1;
-; SM70-NEXT: @%p1 bra $L__BB259_1;
+; SM70-NEXT: @%p1 bra $L__BB342_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
ret <4 x i8> %retval
}
-define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM70-LABEL: add_monotonic_v8i8_global_cta(
+define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: add_acquire_v8i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
; SM70-NEXT: .reg .b16 %rs<25>;
@@ -14175,8 +16810,8 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM70-NEXT: .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
@@ -14190,7 +16825,7 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM70-NEXT: $L__BB260_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB343_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -14248,103 +16883,152 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB260_1;
+; SM70-NEXT: @%p1 bra $L__BB343_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
ret <8 x i8> %retval
}
-define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM70-LABEL: add_acquire_v8i8_global_cta(
+define <1 x i8> @add_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: add_release_v1i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<25>;
-; SM70-NEXT: .reg .b32 %r<35>;
-; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: ld.param.b64 %rd2, [add_release_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r6, [add_release_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB344_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB344_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_release_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB345_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB345_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: add_release_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM70-NEXT: $L__BB261_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB346_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
-; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
-; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
-; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
-; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
-; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
-; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
-; SM70-NEXT: add.s16 %rs18, %rs17, %rs16;
-; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
-; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
-; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
-; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
-; SM70-NEXT: add.s16 %rs21, %rs20, %rs19;
-; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
-; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
-; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
-; SM70-NEXT: add.s16 %rs24, %rs23, %rs22;
-; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
-; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
-; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB261_1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB346_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
- ret <8 x i8> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
}
define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
@@ -14372,7 +17056,7 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM70-NEXT: $L__BB262_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB347_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -14430,7 +17114,7 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB262_1;
+; SM70-NEXT: @%p1 bra $L__BB347_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM70-NEXT: ret;
@@ -14438,6 +17122,148 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i8> @add_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: add_seq_cst_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [add_seq_cst_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.b8 %r6, [add_seq_cst_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB348_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB348_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_seq_cst_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB349_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB349_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: add_seq_cst_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: $L__BB350_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB350_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: add_seq_cst_v8i8_global_cta(
; SM70: {
@@ -14463,7 +17289,7 @@ define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM70-NEXT: $L__BB263_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB351_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -14521,7 +17347,7 @@ define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB263_1;
+; SM70-NEXT: @%p1 bra $L__BB351_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -14530,6 +17356,22 @@ define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i32> @add_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: add_monotonic_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v1i32_global_cta_param_0];
+; SM70-NEXT: ld.param.b32 %r1, [add_monotonic_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: add_monotonic_v2i32_global_cta(
; SM70: {
@@ -14547,151 +17389,197 @@ define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i3
ret <2 x i32> %retval
}
-define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM70-LABEL: add_acquire_v2i32_global_cta(
+define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_monotonic_v4i32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b32 %r<9>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
- ret <2 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
}
-define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM70-LABEL: add_release_v2i32_global_cta(
+define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: add_monotonic_v8i32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b32 %r<17>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @add_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: add_acquire_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v1i32_global_cta_param_0];
+; SM70-NEXT: ld.param.b32 %r1, [add_acquire_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
- ret <2 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
+ ret <1 x i32> %retval
}
-define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM70-LABEL: add_seq_cst_v2i32_global_cta(
+define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: add_acquire_v2i32_global_cta(
; SM70: {
; SM70-NEXT: .reg .b32 %r<5>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
ret <2 x i32> %retval
}
-define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM70-LABEL: add_monotonic_v4i32_global_cta(
+define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_acquire_v4i32_global_cta(
; SM70: {
; SM70-NEXT: .reg .b32 %r<9>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v4i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v4i32_global_cta_param_1];
; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
ret <4 x i32> %retval
}
-define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM70-LABEL: add_acquire_v4i32_global_cta(
+define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: add_acquire_v8i32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b32 %r<17>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v4i32_global_cta_param_0];
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v4i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
- ret <4 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
}
-define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM70-LABEL: add_release_v4i32_global_cta(
+define <1 x i32> @add_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: add_release_v1i32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b32 %r<3>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v1i32_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ld.param.b32 %r1, [add_release_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
- ret <4 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
+ ret <1 x i32> %retval
}
-define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM70-LABEL: add_seq_cst_v4i32_global_cta(
+define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: add_release_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_release_v4i32_global_cta(
; SM70: {
; SM70-NEXT: .reg .b32 %r<9>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
ret <4 x i32> %retval
}
-define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM70-LABEL: add_monotonic_v8i32_global_cta(
+define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: add_release_v8i32_global_cta(
; SM70: {
; SM70-NEXT: .reg .b32 %r<17>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_release_v8i32_global_cta_param_1];
; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
@@ -14703,60 +17591,66 @@ define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i3
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
ret <8 x i32> %retval
}
-define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM70-LABEL: add_acquire_v8i32_global_cta(
+define <1 x i32> @add_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: add_seq_cst_v1i32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b32 %r<3>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.b32 %r1, [add_seq_cst_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
- ret <8 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
+ ret <1 x i32> %retval
}
-define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM70-LABEL: add_release_v8i32_global_cta(
+define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: add_seq_cst_v2i32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b32 %r<5>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_release_v8i32_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v8i32_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_release_v8i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
-; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
- ret <8 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_seq_cst_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
}
define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
@@ -14786,6 +17680,44 @@ define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: nand_monotonic_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [nand_monotonic_v1i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b8 %r6, [nand_monotonic_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: $L__BB368_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
+; SM70-NEXT: @%p1 bra $L__BB368_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
+; SM70-NEXT: st.param.b8 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: nand_monotonic_v2i8_global_cta(
; SM70: {
@@ -14806,7 +17738,7 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM70-NEXT: shl.b32 %r8, %r7, %r1;
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM70-NEXT: $L__BB276_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB369_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r18, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -14824,7 +17756,7 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM70-NEXT: mov.b32 %r18, %r3;
-; SM70-NEXT: @%p1 bra $L__BB276_1;
+; SM70-NEXT: @%p1 bra $L__BB369_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r17, %r3, %r1;
; SM70-NEXT: st.param.b16 [func_retval0], %r17;
@@ -14833,6 +17765,110 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
ret <2 x i8> %retval
}
+define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_monotonic_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB370_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB370_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_monotonic_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB371_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB371_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: nand_acquire_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [nand_acquire_v1i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b8 %r6, [nand_acquire_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: $L__BB372_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
+; SM70-NEXT: @%p1 bra $L__BB372_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: nand_acquire_v2i8_global_cta(
; SM70: {
@@ -14853,7 +17889,7 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: shl.b32 %r8, %r7, %r1;
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM70-NEXT: $L__BB277_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB373_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r18, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -14871,7 +17907,7 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM70-NEXT: mov.b32 %r18, %r3;
-; SM70-NEXT: @%p1 bra $L__BB277_1;
+; SM70-NEXT: @%p1 bra $L__BB373_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r17, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -14881,56 +17917,114 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
ret <2 x i8> %retval
}
-define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM70-LABEL: nand_release_v2i8_global_cta(
+define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_acquire_v4i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<9>;
-; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB374_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB374_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_acquire_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB375_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB375_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: nand_release_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<17>;
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
-; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM70-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd2, [nand_release_v1i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r6, [nand_release_v1i8_global_cta_param_1];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: and.b32 %r6, %r5, 3;
-; SM70-NEXT: shl.b32 %r1, %r6, 3;
-; SM70-NEXT: mov.b32 %r7, 65535;
-; SM70-NEXT: shl.b32 %r8, %r7, %r1;
-; SM70-NEXT: not.b32 %r2, %r8;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM70-NEXT: $L__BB278_1: // %atomicrmw.start
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: $L__BB376_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r9, %r18, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM70-NEXT: and.b32 %r11, %r10, %r4;
-; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM70-NEXT: shl.b32 %r14, %r13, %r1;
-; SM70-NEXT: and.b32 %r15, %r18, %r2;
-; SM70-NEXT: or.b32 %r16, %r15, %r14;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM70-NEXT: mov.b32 %r18, %r3;
-; SM70-NEXT: @%p1 bra $L__BB278_1;
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
+; SM70-NEXT: @%p1 bra $L__BB376_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r17, %r3, %r1;
-; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
+; SM70-NEXT: st.param.b8 [func_retval0], %r15;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
- ret <2 x i8> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
+ ret <1 x i8> %retval
}
-define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM70-LABEL: nand_seq_cst_v2i8_global_cta(
+define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: nand_release_v2i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
; SM70-NEXT: .reg .b16 %rs<9>;
@@ -14938,10 +18032,10 @@ define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM70-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: and.b32 %r6, %r5, 3;
@@ -14950,7 +18044,7 @@ define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: shl.b32 %r8, %r7, %r1;
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM70-NEXT: $L__BB279_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB377_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r18, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -14968,69 +18062,15 @@ define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM70-NEXT: mov.b32 %r18, %r3;
-; SM70-NEXT: @%p1 bra $L__BB279_1;
+; SM70-NEXT: @%p1 bra $L__BB377_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r17, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b16 [func_retval0], %r17;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
ret <2 x i8> %retval
}
-define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM70-LABEL: nand_monotonic_v4i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<6>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB280_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b32 %r3, %r5, %r2;
-; SM70-NEXT: xor.b32 %r4, %r3, -1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
-; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
-; SM70-NEXT: mov.b32 %r5, %r1;
-; SM70-NEXT: @%p1 bra $L__BB280_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
- ret <4 x i8> %retval
-}
-
-define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM70-LABEL: nand_acquire_v4i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<6>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB281_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b32 %r3, %r5, %r2;
-; SM70-NEXT: xor.b32 %r4, %r3, -1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
-; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
-; SM70-NEXT: mov.b32 %r5, %r1;
-; SM70-NEXT: @%p1 bra $L__BB281_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
- ret <4 x i8> %retval
-}
-
define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: nand_release_v4i8_global_cta(
; SM70: {
@@ -15043,14 +18083,14 @@ define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB282_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB378_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r3, %r5, %r2;
; SM70-NEXT: xor.b32 %r4, %r3, -1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM70-NEXT: mov.b32 %r5, %r1;
-; SM70-NEXT: @%p1 bra $L__BB282_1;
+; SM70-NEXT: @%p1 bra $L__BB378_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NEXT: ret;
@@ -15058,48 +18098,21 @@ define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
ret <4 x i8> %retval
}
-define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM70-LABEL: nand_seq_cst_v4i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<6>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [nand_seq_cst_v4i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB283_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b32 %r3, %r5, %r2;
-; SM70-NEXT: xor.b32 %r4, %r3, -1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
-; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
-; SM70-NEXT: mov.b32 %r5, %r1;
-; SM70-NEXT: @%p1 bra $L__BB283_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
- ret <4 x i8> %retval
-}
-
-define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM70-LABEL: nand_monotonic_v8i8_global_cta(
+define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_release_v8i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
; SM70-NEXT: .reg .b32 %r<9>;
; SM70-NEXT: .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: $L__BB284_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB379_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -15117,488 +18130,250 @@ define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB284_1;
+; SM70-NEXT: @%p1 bra $L__BB379_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
ret <8 x i8> %retval
}
-define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM70-LABEL: nand_acquire_v8i8_global_cta(
+define <1 x i8> @nand_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: nand_seq_cst_v1i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<9>;
-; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.b8 %r6, [nand_seq_cst_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: $L__BB285_1: // %atomicrmw.start
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: $L__BB380_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r3, %r8, %r2;
-; SM70-NEXT: and.b32 %r4, %r7, %r1;
-; SM70-NEXT: xor.b32 %r5, %r4, -1;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM70-NEXT: xor.b32 %r6, %r3, -1;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
-; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB285_1;
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
+; SM70-NEXT: @%p1 bra $L__BB380_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: st.param.b8 [func_retval0], %r15;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
- ret <8 x i8> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
+ ret <1 x i8> %retval
}
-define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM70-LABEL: nand_release_v8i8_global_cta(
+define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: nand_seq_cst_v2i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<9>;
-; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: $L__BB286_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r3, %r8, %r2;
-; SM70-NEXT: and.b32 %r4, %r7, %r1;
-; SM70-NEXT: xor.b32 %r5, %r4, -1;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM70-NEXT: xor.b32 %r6, %r3, -1;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
-; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB286_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
- ret <8 x i8> %retval
-}
-
-define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM70-LABEL: nand_seq_cst_v8i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<9>;
-; SM70-NEXT: .reg .b64 %rd<12>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: $L__BB287_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r3, %r8, %r2;
-; SM70-NEXT: and.b32 %r4, %r7, %r1;
-; SM70-NEXT: xor.b32 %r5, %r4, -1;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM70-NEXT: xor.b32 %r6, %r3, -1;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
-; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB287_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
- ret <8 x i8> %retval
-}
-
-define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM70-LABEL: nand_monotonic_v2i32_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<7>;
-; SM70-NEXT: .reg .b64 %rd<13>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB288_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r3, %r6, %r2;
-; SM70-NEXT: and.b32 %r4, %r5, %r1;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: not.b64 %rd11, %rd10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB288_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
- ret <2 x i32> %retval
-}
-
-define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM70-LABEL: nand_acquire_v2i32_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<7>;
-; SM70-NEXT: .reg .b64 %rd<13>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB289_1: // %atomicrmw.start
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: $L__BB381_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r3, %r6, %r2;
-; SM70-NEXT: and.b32 %r4, %r5, %r1;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: not.b64 %rd11, %rd10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB289_1;
+; SM70-NEXT: shr.u32 %r9, %r18, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, %r4;
+; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM70-NEXT: shl.b32 %r14, %r13, %r1;
+; SM70-NEXT: and.b32 %r15, %r18, %r2;
+; SM70-NEXT: or.b32 %r16, %r15, %r14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: @%p1 bra $L__BB381_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r17, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
- ret <2 x i32> %retval
-}
-
-define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM70-LABEL: nand_release_v2i32_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<7>;
-; SM70-NEXT: .reg .b64 %rd<13>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB290_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r3, %r6, %r2;
-; SM70-NEXT: and.b32 %r4, %r5, %r1;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: not.b64 %rd11, %rd10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB290_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: st.param.b16 [func_retval0], %r17;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
- ret <2 x i32> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
}
-define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM70-LABEL: nand_seq_cst_v2i32_global_cta(
+define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_seq_cst_v4i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<7>;
-; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
+; SM70-NEXT: ld.param.b32 %r2, [nand_seq_cst_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB291_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r3, %r6, %r2;
-; SM70-NEXT: and.b32 %r4, %r5, %r1;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: not.b64 %rd11, %rd10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB291_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
- ret <2 x i32> %retval
-}
-
-define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM70-LABEL: nand_monotonic_v4i32_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<3>;
-; SM70-NEXT: .reg .b32 %r<13>;
-; SM70-NEXT: .reg .b64 %rd<24>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB292_1: // %atomicrmw.start
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB382_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r5, %r10, %r2;
-; SM70-NEXT: and.b32 %r6, %r9, %r1;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: not.b64 %rd11, %rd10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB292_1;
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB382_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
-; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM70-NEXT: $L__BB292_3: // %atomicrmw.start2
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
-; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
-; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
-; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
-; SM70-NEXT: and.b32 %r7, %r12, %r4;
-; SM70-NEXT: and.b32 %r8, %r11, %r3;
-; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
-; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
-; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM70-NEXT: not.b64 %rd22, %rd21;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
-; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
-; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB292_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end1
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
- ret <4 x i32> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
}
-define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM70-LABEL: nand_acquire_v4i32_global_cta(
+define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_seq_cst_v8i8_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<3>;
-; SM70-NEXT: .reg .b32 %r<13>;
-; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB293_1: // %atomicrmw.start
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB383_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r5, %r10, %r2;
-; SM70-NEXT: and.b32 %r6, %r9, %r1;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: not.b64 %rd11, %rd10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB293_1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB383_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
-; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM70-NEXT: $L__BB293_3: // %atomicrmw.start2
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
-; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
-; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
-; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
-; SM70-NEXT: and.b32 %r7, %r12, %r4;
-; SM70-NEXT: and.b32 %r8, %r11, %r3;
-; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
-; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
-; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM70-NEXT: not.b64 %rd22, %rd21;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
-; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
-; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB293_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
- ret <4 x i32> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
}
-define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM70-LABEL: nand_release_v4i32_global_cta(
+define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: nand_monotonic_v1i32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<3>;
-; SM70-NEXT: .reg .b32 %r<13>;
-; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
+; SM70-NEXT: ld.param.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB384_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: not.b32 %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p1 bra $L__BB384_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_monotonic_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB294_1: // %atomicrmw.start
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB385_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r5, %r10, %r2;
-; SM70-NEXT: and.b32 %r6, %r9, %r1;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: not.b64 %rd11, %rd10;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB294_1;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB385_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
-; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM70-NEXT: $L__BB294_3: // %atomicrmw.start2
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
-; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
-; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
-; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
-; SM70-NEXT: and.b32 %r7, %r12, %r4;
-; SM70-NEXT: and.b32 %r8, %r11, %r3;
-; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
-; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
-; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM70-NEXT: not.b64 %rd22, %rd21;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
-; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
-; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB294_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end1
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
- ret <4 x i32> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
}
-define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM70-LABEL: nand_seq_cst_v4i32_global_cta(
+define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_monotonic_v4i32_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<3>;
; SM70-NEXT: .reg .b32 %r<13>;
; SM70-NEXT: .reg .b64 %rd<24>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB295_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB386_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -15615,12 +18390,12 @@ define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB295_1;
+; SM70-NEXT: @%p1 bra $L__BB386_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM70-NEXT: $L__BB295_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB386_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
@@ -15637,12 +18412,11 @@ define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB295_3;
+; SM70-NEXT: @%p2 bra $L__BB386_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
ret <4 x i32> %retval
}
@@ -15660,7 +18434,7 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM70-NEXT: $L__BB296_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB387_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -15677,12 +18451,12 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB296_1;
+; SM70-NEXT: @%p1 bra $L__BB387_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM70-NEXT: $L__BB296_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB387_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -15699,12 +18473,12 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB296_3;
+; SM70-NEXT: @%p2 bra $L__BB387_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM70-NEXT: $L__BB296_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB387_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -15721,12 +18495,12 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM70-NEXT: @%p3 bra $L__BB296_5;
+; SM70-NEXT: @%p3 bra $L__BB387_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM70-NEXT: $L__BB296_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB387_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -15743,13 +18517,140 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM70-NEXT: @%p4 bra $L__BB296_7;
+; SM70-NEXT: @%p4 bra $L__BB387_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
- ret <8 x i32> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: nand_acquire_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
+; SM70-NEXT: ld.param.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB388_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: not.b32 %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p1 bra $L__BB388_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_acquire_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB389_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB389_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_acquire_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB390_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r10, %r2;
+; SM70-NEXT: and.b32 %r6, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB390_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM70-NEXT: $L__BB390_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r7, %r12, %r4;
+; SM70-NEXT: and.b32 %r8, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB390_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
}
define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
@@ -15766,7 +18667,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM70-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB391_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -15783,12 +18684,12 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB297_1;
+; SM70-NEXT: @%p1 bra $L__BB391_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM70-NEXT: $L__BB297_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB391_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -15805,12 +18706,12 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB297_3;
+; SM70-NEXT: @%p2 bra $L__BB391_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM70-NEXT: $L__BB297_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB391_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -15827,12 +18728,12 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM70-NEXT: @%p3 bra $L__BB297_5;
+; SM70-NEXT: @%p3 bra $L__BB391_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM70-NEXT: $L__BB297_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB391_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -15849,7 +18750,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM70-NEXT: @%p4 bra $L__BB297_7;
+; SM70-NEXT: @%p4 bra $L__BB391_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -15859,6 +18760,133 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: nand_release_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [nand_release_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB392_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: not.b32 %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p1 bra $L__BB392_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_release_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB393_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB393_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_release_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB394_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r10, %r2;
+; SM70-NEXT: and.b32 %r6, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB394_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM70-NEXT: $L__BB394_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r7, %r12, %r4;
+; SM70-NEXT: and.b32 %r8, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB394_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: nand_release_v8i32_global_cta(
; SM70: {
@@ -15874,7 +18902,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM70-NEXT: $L__BB298_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB395_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -15891,12 +18919,12 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB298_1;
+; SM70-NEXT: @%p1 bra $L__BB395_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM70-NEXT: $L__BB298_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB395_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -15913,12 +18941,12 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB298_3;
+; SM70-NEXT: @%p2 bra $L__BB395_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM70-NEXT: $L__BB298_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB395_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -15935,12 +18963,12 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM70-NEXT: @%p3 bra $L__BB298_5;
+; SM70-NEXT: @%p3 bra $L__BB395_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM70-NEXT: $L__BB298_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB395_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -15957,7 +18985,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM70-NEXT: @%p4 bra $L__BB298_7;
+; SM70-NEXT: @%p4 bra $L__BB395_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
@@ -15966,6 +18994,136 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i32> @nand_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: nand_seq_cst_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.b32 %r1, [nand_seq_cst_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB396_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: not.b32 %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p1 bra $L__BB396_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_seq_cst_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB397_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB397_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_seq_cst_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB398_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r10, %r2;
+; SM70-NEXT: and.b32 %r6, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB398_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM70-NEXT: $L__BB398_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r7, %r12, %r4;
+; SM70-NEXT: and.b32 %r8, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB398_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: nand_seq_cst_v8i32_global_cta(
; SM70: {
@@ -15981,7 +19139,7 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM70-NEXT: $L__BB299_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB399_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -15998,12 +19156,12 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB299_1;
+; SM70-NEXT: @%p1 bra $L__BB399_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM70-NEXT: $L__BB299_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB399_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -16020,12 +19178,12 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB299_3;
+; SM70-NEXT: @%p2 bra $L__BB399_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM70-NEXT: $L__BB299_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB399_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -16042,12 +19200,12 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM70-NEXT: @%p3 bra $L__BB299_5;
+; SM70-NEXT: @%p3 bra $L__BB399_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM70-NEXT: $L__BB299_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB399_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -16064,7 +19222,7 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM70-NEXT: @%p4 bra $L__BB299_7;
+; SM70-NEXT: @%p4 bra $L__BB399_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
index a868ebffa4f9c..18b284a04fc48 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
@@ -2,6 +2,43 @@
; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefix=SM90
; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %}
+define <1 x i8> @xchg_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: xchg_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<14>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r5, [xchg_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
+; SM90-NEXT: mov.b32 %r8, 255;
+; SM90-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-NEXT: not.b32 %r2, %r9;
+; SM90-NEXT: shl.b32 %r3, %r5, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-NEXT: $L__BB0_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-NEXT: or.b32 %r11, %r10, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM90-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM90-NEXT: mov.b32 %r13, %r4;
+; SM90-NEXT: @%p1 bra $L__BB0_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r12, %r4, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r12;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: xchg_acq_rel_v2i8_global_cta(
; SM90: {
@@ -22,14 +59,14 @@ define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: not.b32 %r2, %r9;
; SM90-NEXT: shl.b32 %r3, %r5, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
-; SM90-NEXT: $L__BB0_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB1_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r10, %r13, %r2;
; SM90-NEXT: or.b32 %r11, %r10, %r3;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
; SM90-NEXT: setp.ne.b32 %p1, %r4, %r13;
; SM90-NEXT: mov.b32 %r13, %r4;
-; SM90-NEXT: @%p1 bra $L__BB0_1;
+; SM90-NEXT: @%p1 bra $L__BB1_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r12, %r4, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -70,6 +107,43 @@ define <8 x i8> @xchg_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
ret <8 x i8> %retval
}
+define <1 x i16> @xchg_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: xchg_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<14>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
+; SM90-NEXT: mov.b32 %r8, 65535;
+; SM90-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-NEXT: not.b32 %r2, %r9;
+; SM90-NEXT: shl.b32 %r3, %r5, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-NEXT: $L__BB4_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-NEXT: or.b32 %r11, %r10, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM90-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM90-NEXT: mov.b32 %r13, %r4;
+; SM90-NEXT: @%p1 bra $L__BB4_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r12, %r4, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r12;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @xchg_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: xchg_acq_rel_v2i16_global_cta(
; SM90: {
@@ -121,6 +195,24 @@ define <8 x i16> @xchg_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
ret <8 x i16> %retval
}
+define <1 x i32> @xchg_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: xchg_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @xchg_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: xchg_acq_rel_v2i32_global_cta(
; SM90: {
@@ -188,6 +280,23 @@ define <8 x i32> @xchg_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i64> @xchg_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: xchg_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @xchg_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: xchg_acq_rel_v2i64_global_cta(
; SM90: {
@@ -258,6 +367,45 @@ define <8 x i64> @xchg_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
ret <8 x i64> %retval
}
+define <1 x i8> @add_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: add_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r6, [add_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB16_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB16_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: add_acq_rel_v2i8_global_cta(
; SM90: {
@@ -279,7 +427,7 @@ define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: shl.b32 %r8, %r7, %r1;
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM90-NEXT: $L__BB12_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB17_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r17, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -297,7 +445,7 @@ define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
; SM90-NEXT: mov.b32 %r17, %r3;
-; SM90-NEXT: @%p1 bra $L__BB12_1;
+; SM90-NEXT: @%p1 bra $L__BB17_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r16, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -328,7 +476,7 @@ define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: $L__BB13_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB18_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -352,7 +500,7 @@ define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM90-NEXT: mov.b32 %r18, %r1;
-; SM90-NEXT: @%p1 bra $L__BB13_1;
+; SM90-NEXT: @%p1 bra $L__BB18_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -386,7 +534,7 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM90-NEXT: $L__BB14_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB19_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -444,7 +592,7 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB14_1;
+; SM90-NEXT: @%p1 bra $L__BB19_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -453,6 +601,45 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @add_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: add_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r6, [add_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 65535;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB20_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB20_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: add_acq_rel_v2i16_global_cta(
; SM90: {
@@ -465,13 +652,13 @@ define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB15_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB21_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB15_1;
+; SM90-NEXT: @%p1 bra $L__BB21_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -494,7 +681,7 @@ define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB16_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB22_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s16x2 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -510,7 +697,7 @@ define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB16_1;
+; SM90-NEXT: @%p1 bra $L__BB22_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -542,7 +729,7 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB17_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB23_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s16x2 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -579,7 +766,7 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB17_1;
+; SM90-NEXT: @%p1 bra $L__BB23_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -588,6 +775,24 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @add_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: add_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [add_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @add_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: add_acq_rel_v2i32_global_cta(
; SM90: {
@@ -655,6 +860,23 @@ define <8 x i32> @add_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @add_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: add_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @add_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: add_acq_rel_v2i64_global_cta(
; SM90: {
@@ -725,6 +947,45 @@ define <8 x i64> @add_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @sub_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: sub_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r6, [sub_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB32_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB32_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: sub_acq_rel_v2i8_global_cta(
; SM90: {
@@ -747,7 +1008,7 @@ define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM90-NEXT: $L__BB24_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB33_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r15, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -764,7 +1025,7 @@ define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r15;
; SM90-NEXT: mov.b32 %r15, %r3;
-; SM90-NEXT: @%p1 bra $L__BB24_1;
+; SM90-NEXT: @%p1 bra $L__BB33_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -795,7 +1056,7 @@ define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: $L__BB25_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB34_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -819,7 +1080,7 @@ define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM90-NEXT: mov.b32 %r18, %r1;
-; SM90-NEXT: @%p1 bra $L__BB25_1;
+; SM90-NEXT: @%p1 bra $L__BB34_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -853,7 +1114,7 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM90-NEXT: $L__BB26_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB35_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -911,7 +1172,7 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB26_1;
+; SM90-NEXT: @%p1 bra $L__BB35_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -920,6 +1181,45 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @sub_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: sub_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r6, [sub_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 65535;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB36_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB36_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: sub_acq_rel_v2i16_global_cta(
; SM90: {
@@ -934,7 +1234,7 @@ define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NEXT: $L__BB27_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB37_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM90-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -943,7 +1243,7 @@ define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB27_1;
+; SM90-NEXT: @%p1 bra $L__BB37_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -969,7 +1269,7 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM90-NEXT: $L__BB28_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB38_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r5;
; SM90-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -991,7 +1291,7 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB28_1;
+; SM90-NEXT: @%p1 bra $L__BB38_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -1026,7 +1326,7 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM90-NEXT: $L__BB29_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB39_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r9;
; SM90-NEXT: sub.s16 %rs5, %rs4, %rs2;
@@ -1077,7 +1377,7 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB29_1;
+; SM90-NEXT: @%p1 bra $L__BB39_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -1086,6 +1386,25 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @sub_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: sub_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<4>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [sub_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: neg.s32 %r2, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: sub_acq_rel_v2i32_global_cta(
; SM90: {
@@ -1167,6 +1486,24 @@ define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @sub_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: sub_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<5>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: neg.s64 %rd3, %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd4;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: sub_acq_rel_v2i64_global_cta(
; SM90: {
@@ -1251,6 +1588,34 @@ define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @and_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: and_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<12>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r1, [and_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM90-NEXT: and.b32 %r3, %r2, 3;
+; SM90-NEXT: shl.b32 %r4, %r3, 3;
+; SM90-NEXT: mov.b32 %r5, 255;
+; SM90-NEXT: shl.b32 %r6, %r5, %r4;
+; SM90-NEXT: not.b32 %r7, %r6;
+; SM90-NEXT: shl.b32 %r8, %r1, %r4;
+; SM90-NEXT: or.b32 %r9, %r8, %r7;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM90-NEXT: shr.u32 %r11, %r10, %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r11;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @and_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: and_acq_rel_v2i8_global_cta(
; SM90: {
@@ -1291,13 +1656,13 @@ define <4 x i8> @and_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB37_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB50_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB37_1;
+; SM90-NEXT: @%p1 bra $L__BB50_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -1320,7 +1685,7 @@ define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB38_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB51_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -1336,7 +1701,7 @@ define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB38_1;
+; SM90-NEXT: @%p1 bra $L__BB51_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -1345,6 +1710,34 @@ define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @and_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: and_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<12>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r1, [and_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM90-NEXT: and.b32 %r3, %r2, 3;
+; SM90-NEXT: shl.b32 %r4, %r3, 3;
+; SM90-NEXT: mov.b32 %r5, 65535;
+; SM90-NEXT: shl.b32 %r6, %r5, %r4;
+; SM90-NEXT: not.b32 %r7, %r6;
+; SM90-NEXT: shl.b32 %r8, %r1, %r4;
+; SM90-NEXT: or.b32 %r9, %r8, %r7;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM90-NEXT: shr.u32 %r11, %r10, %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r11;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: and_acq_rel_v2i16_global_cta(
; SM90: {
@@ -1357,13 +1750,13 @@ define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB39_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB53_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB39_1;
+; SM90-NEXT: @%p1 bra $L__BB53_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -1386,7 +1779,7 @@ define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB40_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB54_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -1402,7 +1795,7 @@ define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB40_1;
+; SM90-NEXT: @%p1 bra $L__BB54_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -1434,7 +1827,7 @@ define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB41_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB55_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -1471,7 +1864,7 @@ define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB41_1;
+; SM90-NEXT: @%p1 bra $L__BB55_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -1480,6 +1873,24 @@ define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @and_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: and_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [and_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @and_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: and_acq_rel_v2i32_global_cta(
; SM90: {
@@ -1547,7 +1958,24 @@ define <8 x i32> @and_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
-define <2 x i64> @and_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+define <1 x i64> @and_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: and_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [and_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @and_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: and_acq_rel_v2i64_global_cta(
; SM90: {
; SM90-NEXT: .reg .b64 %rd<6>;
@@ -1617,6 +2045,46 @@ define <8 x i64> @and_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @nand_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: nand_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r6, [nand_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM90-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
+; SM90-NEXT: @%p1 bra $L__BB64_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r15;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: nand_acq_rel_v2i8_global_cta(
; SM90: {
@@ -1638,7 +2106,7 @@ define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: shl.b32 %r8, %r7, %r1;
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: $L__BB48_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB65_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r18, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -1656,7 +2124,7 @@ define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM90-NEXT: mov.b32 %r18, %r3;
-; SM90-NEXT: @%p1 bra $L__BB48_1;
+; SM90-NEXT: @%p1 bra $L__BB65_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -1678,14 +2146,14 @@ define <4 x i8> @nand_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB49_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB66_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r5, %r2;
; SM90-NEXT: xor.b32 %r4, %r3, -1;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM90-NEXT: mov.b32 %r5, %r1;
-; SM90-NEXT: @%p1 bra $L__BB49_1;
+; SM90-NEXT: @%p1 bra $L__BB66_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -1708,7 +2176,7 @@ define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: $L__BB50_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB67_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -1726,7 +2194,7 @@ define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB50_1;
+; SM90-NEXT: @%p1 bra $L__BB67_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -1735,6 +2203,46 @@ define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
ret <8 x i8> %retval
}
+define <1 x i16> @nand_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: nand_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r6, [nand_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 65535;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM90-NEXT: $L__BB68_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
+; SM90-NEXT: @%p1 bra $L__BB68_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r15;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: nand_acq_rel_v2i16_global_cta(
; SM90: {
@@ -1747,14 +2255,14 @@ define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB51_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB69_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r5, %r2;
; SM90-NEXT: xor.b32 %r4, %r3, -1;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM90-NEXT: mov.b32 %r5, %r1;
-; SM90-NEXT: @%p1 bra $L__BB51_1;
+; SM90-NEXT: @%p1 bra $L__BB69_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -1777,7 +2285,7 @@ define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: $L__BB52_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB70_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -1795,7 +2303,7 @@ define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB52_1;
+; SM90-NEXT: @%p1 bra $L__BB70_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -1827,7 +2335,7 @@ define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM90-NEXT: cvt.u32.u64 %r15, %rd4;
; SM90-NEXT: mov.b64 {_, %r14}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r13, %rd3;
-; SM90-NEXT: $L__BB53_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB71_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r13;
; SM90-NEXT: cvt.u64.u32 %rd6, %r14;
@@ -1868,7 +2376,7 @@ define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM90-NEXT: mov.b64 {_, %r14}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r13, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB53_1;
+; SM90-NEXT: @%p1 bra $L__BB71_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
@@ -1877,6 +2385,34 @@ define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
ret <8 x i16> %retval
}
+define <1 x i32> @nand_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: nand_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [nand_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: not.b32 %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p1 bra $L__BB72_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: nand_acq_rel_v2i32_global_cta(
; SM90: {
@@ -1891,7 +2427,7 @@ define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB54_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB73_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -1908,7 +2444,7 @@ define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM90-NEXT: @%p1 bra $L__BB54_1;
+; SM90-NEXT: @%p1 bra $L__BB73_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -1940,7 +2476,7 @@ define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB55_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB74_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
@@ -1979,7 +2515,7 @@ define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB55_1;
+; SM90-NEXT: @%p1 bra $L__BB74_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -2012,7 +2548,7 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB56_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB75_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
@@ -2051,7 +2587,7 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB56_1;
+; SM90-NEXT: @%p1 bra $L__BB75_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd28, 0;
; SM90-NEXT: {
@@ -2065,7 +2601,7 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
-; SM90-NEXT: $L__BB56_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB75_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
@@ -2104,7 +2640,7 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
-; SM90-NEXT: @%p2 bra $L__BB56_3;
+; SM90-NEXT: @%p2 bra $L__BB75_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -2114,6 +2650,33 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i64> @nand_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: nand_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<7>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM90-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b64 %rd4, %rd6, %rd1;
+; SM90-NEXT: not.b64 %rd5, %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
+; SM90-NEXT: mov.b64 %rd6, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB76_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: nand_acq_rel_v2i64_global_cta(
; SM90: {
@@ -2132,7 +2695,7 @@ define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
; SM90-NEXT: mov.b128 {%rd14, %rd15}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB57_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB77_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b64 %rd7, %rd15, %rd5;
; SM90-NEXT: and.b64 %rd8, %rd14, %rd4;
@@ -2151,7 +2714,7 @@ define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64
; SM90-NEXT: setp.ne.b64 %p1, %rd13, 0;
; SM90-NEXT: mov.b64 %rd14, %rd1;
; SM90-NEXT: mov.b64 %rd15, %rd2;
-; SM90-NEXT: @%p1 bra $L__BB57_1;
+; SM90-NEXT: @%p1 bra $L__BB77_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
@@ -2179,7 +2742,7 @@ define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
; SM90-NEXT: mov.b128 {%rd26, %rd27}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB58_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB78_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b64 %rd11, %rd27, %rd2;
; SM90-NEXT: and.b64 %rd12, %rd26, %rd1;
@@ -2198,7 +2761,7 @@ define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64
; SM90-NEXT: setp.ne.b64 %p1, %rd17, 0;
; SM90-NEXT: mov.b64 %rd26, %rd5;
; SM90-NEXT: mov.b64 %rd27, %rd6;
-; SM90-NEXT: @%p1 bra $L__BB58_1;
+; SM90-NEXT: @%p1 bra $L__BB78_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
@@ -2208,7 +2771,7 @@ define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd28, %rd29}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB58_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB78_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b64 %rd19, %rd29, %rd4;
; SM90-NEXT: and.b64 %rd20, %rd28, %rd3;
@@ -2227,7 +2790,7 @@ define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64
; SM90-NEXT: setp.ne.b64 %p2, %rd25, 0;
; SM90-NEXT: mov.b64 %rd28, %rd7;
; SM90-NEXT: mov.b64 %rd29, %rd8;
-; SM90-NEXT: @%p2 bra $L__BB58_3;
+; SM90-NEXT: @%p2 bra $L__BB78_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -2258,7 +2821,7 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB59_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB79_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd10, %rd51;
; SM90-NEXT: mov.b64 %rd9, %rd50;
@@ -2277,7 +2840,7 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: xor.b64 %rd24, %rd50, %rd9;
; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB59_1;
+; SM90-NEXT: @%p1 bra $L__BB79_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd26, 0;
; SM90-NEXT: {
@@ -2287,7 +2850,7 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB59_3: // %atomicrmw.start6
+; SM90-NEXT: $L__BB79_3: // %atomicrmw.start6
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd12, %rd53;
; SM90-NEXT: mov.b64 %rd11, %rd52;
@@ -2306,7 +2869,7 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: xor.b64 %rd32, %rd52, %rd11;
; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
; SM90-NEXT: setp.ne.b64 %p2, %rd33, 0;
-; SM90-NEXT: @%p2 bra $L__BB59_3;
+; SM90-NEXT: @%p2 bra $L__BB79_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end5
; SM90-NEXT: mov.b64 %rd34, 0;
; SM90-NEXT: {
@@ -2316,7 +2879,7 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
; SM90-NEXT: mov.b128 {%rd54, %rd55}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB59_5: // %atomicrmw.start16
+; SM90-NEXT: $L__BB79_5: // %atomicrmw.start16
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b64 %rd35, %rd55, %rd2;
; SM90-NEXT: and.b64 %rd36, %rd54, %rd1;
@@ -2335,7 +2898,7 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: setp.ne.b64 %p3, %rd41, 0;
; SM90-NEXT: mov.b64 %rd54, %rd13;
; SM90-NEXT: mov.b64 %rd55, %rd14;
-; SM90-NEXT: @%p3 bra $L__BB59_5;
+; SM90-NEXT: @%p3 bra $L__BB79_5;
; SM90-NEXT: // %bb.6: // %atomicrmw.end15
; SM90-NEXT: mov.b64 %rd42, 0;
; SM90-NEXT: {
@@ -2345,7 +2908,7 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
; SM90-NEXT: mov.b128 {%rd56, %rd57}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB59_7: // %atomicrmw.start22
+; SM90-NEXT: $L__BB79_7: // %atomicrmw.start22
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b64 %rd43, %rd57, %rd4;
; SM90-NEXT: and.b64 %rd44, %rd56, %rd3;
@@ -2364,7 +2927,7 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: setp.ne.b64 %p4, %rd49, 0;
; SM90-NEXT: mov.b64 %rd56, %rd15;
; SM90-NEXT: mov.b64 %rd57, %rd16;
-; SM90-NEXT: @%p4 bra $L__BB59_7;
+; SM90-NEXT: @%p4 bra $L__BB79_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end21
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -2376,6 +2939,30 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
ret <8 x i64> %retval
}
+define <1 x i8> @or_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: or_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<8>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r1, [or_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM90-NEXT: and.b32 %r3, %r2, 3;
+; SM90-NEXT: shl.b32 %r4, %r3, 3;
+; SM90-NEXT: shl.b32 %r5, %r1, %r4;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM90-NEXT: shr.u32 %r7, %r6, %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r7;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @or_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: or_acq_rel_v2i8_global_cta(
; SM90: {
@@ -2412,13 +2999,13 @@ define <4 x i8> @or_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %va
; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB82_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: or.b32 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB61_1;
+; SM90-NEXT: @%p1 bra $L__BB82_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -2441,7 +3028,7 @@ define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %va
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB83_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: or.b32 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -2457,7 +3044,7 @@ define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %va
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB62_1;
+; SM90-NEXT: @%p1 bra $L__BB83_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -2466,6 +3053,30 @@ define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %va
ret <8 x i8> %retval
}
+define <1 x i16> @or_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: or_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<8>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r1, [or_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM90-NEXT: and.b32 %r3, %r2, 3;
+; SM90-NEXT: shl.b32 %r4, %r3, 3;
+; SM90-NEXT: shl.b32 %r5, %r1, %r4;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM90-NEXT: shr.u32 %r7, %r6, %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r7;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: or_acq_rel_v2i16_global_cta(
; SM90: {
@@ -2478,13 +3089,13 @@ define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB85_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: or.b32 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB63_1;
+; SM90-NEXT: @%p1 bra $L__BB85_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -2507,7 +3118,7 @@ define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB86_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: or.b32 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -2523,7 +3134,7 @@ define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB64_1;
+; SM90-NEXT: @%p1 bra $L__BB86_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -2555,7 +3166,7 @@ define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB87_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: or.b32 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -2592,7 +3203,7 @@ define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB65_1;
+; SM90-NEXT: @%p1 bra $L__BB87_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -2601,6 +3212,24 @@ define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @or_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: or_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [or_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @or_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: or_acq_rel_v2i32_global_cta(
; SM90: {
@@ -2668,6 +3297,23 @@ define <8 x i32> @or_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @or_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: or_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [or_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @or_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: or_acq_rel_v2i64_global_cta(
; SM90: {
@@ -2738,6 +3384,30 @@ define <8 x i64> @or_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @xor_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: xor_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<8>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r1, [xor_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM90-NEXT: and.b32 %r3, %r2, 3;
+; SM90-NEXT: shl.b32 %r4, %r3, 3;
+; SM90-NEXT: shl.b32 %r5, %r1, %r4;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM90-NEXT: shr.u32 %r7, %r6, %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r7;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @xor_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: xor_acq_rel_v2i8_global_cta(
; SM90: {
@@ -2774,13 +3444,13 @@ define <4 x i8> @xor_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB98_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: xor.b32 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB73_1;
+; SM90-NEXT: @%p1 bra $L__BB98_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -2803,7 +3473,7 @@ define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB99_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: xor.b32 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -2819,7 +3489,7 @@ define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB74_1;
+; SM90-NEXT: @%p1 bra $L__BB99_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -2828,6 +3498,30 @@ define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @xor_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: xor_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<8>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r1, [xor_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM90-NEXT: and.b32 %r3, %r2, 3;
+; SM90-NEXT: shl.b32 %r4, %r3, 3;
+; SM90-NEXT: shl.b32 %r5, %r1, %r4;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM90-NEXT: shr.u32 %r7, %r6, %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r7;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: xor_acq_rel_v2i16_global_cta(
; SM90: {
@@ -2840,13 +3534,13 @@ define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB101_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: xor.b32 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB75_1;
+; SM90-NEXT: @%p1 bra $L__BB101_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -2869,7 +3563,7 @@ define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB102_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: xor.b32 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -2885,7 +3579,7 @@ define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB76_1;
+; SM90-NEXT: @%p1 bra $L__BB102_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -2917,7 +3611,7 @@ define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB103_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: xor.b32 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -2954,7 +3648,7 @@ define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB77_1;
+; SM90-NEXT: @%p1 bra $L__BB103_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -2963,6 +3657,24 @@ define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @xor_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: xor_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [xor_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @xor_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: xor_acq_rel_v2i32_global_cta(
; SM90: {
@@ -3030,6 +3742,23 @@ define <8 x i32> @xor_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @xor_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: xor_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [xor_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @xor_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: xor_acq_rel_v2i64_global_cta(
; SM90: {
@@ -3100,6 +3829,49 @@ define <8 x i64> @xor_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: max_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.s8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB112_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r15, %r1;
+; SM90-NEXT: cvt.s8.s32 %rs2, %r8;
+; SM90-NEXT: max.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: and.b32 %r10, %r9, 255;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p1 bra $L__BB112_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: max_acq_rel_v2i8_global_cta(
; SM90: {
@@ -3125,7 +3897,7 @@ define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: cvt.s16.s8 %rs7, %rs6;
; SM90-NEXT: cvt.s16.s8 %rs8, %rs5;
; SM90-NEXT: mov.b32 %r12, {%rs8, %rs7};
-; SM90-NEXT: $L__BB84_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB113_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r19, %r1;
; SM90-NEXT: cvt.s8.s32 %rs3, %r9;
@@ -3144,7 +3916,7 @@ define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r19;
; SM90-NEXT: mov.b32 %r19, %r3;
-; SM90-NEXT: @%p1 bra $L__BB84_1;
+; SM90-NEXT: @%p1 bra $L__BB113_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r18, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -3170,7 +3942,7 @@ define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
-; SM90-NEXT: $L__BB85_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB114_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
; SM90-NEXT: setp.gt.s32 %p1, %r4, %r3;
@@ -3198,7 +3970,7 @@ define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM90-NEXT: setp.ne.b32 %p5, %r1, %r26;
; SM90-NEXT: mov.b32 %r26, %r1;
-; SM90-NEXT: @%p5 bra $L__BB85_1;
+; SM90-NEXT: @%p5 bra $L__BB114_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -3224,7 +3996,7 @@ define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: cvt.u32.u64 %r49, %rd2;
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
-; SM90-NEXT: $L__BB86_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB115_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r49;
; SM90-NEXT: cvt.u64.u32 %rd4, %r50;
@@ -3314,7 +4086,7 @@ define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r50}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM90-NEXT: @%p9 bra $L__BB86_1;
+; SM90-NEXT: @%p9 bra $L__BB115_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
@@ -3323,6 +4095,48 @@ define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @max_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: max_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB116_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB116_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: max_acq_rel_v2i16_global_cta(
; SM90: {
@@ -3335,13 +4149,13 @@ define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB87_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB117_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.s16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB87_1;
+; SM90-NEXT: @%p1 bra $L__BB117_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -3364,7 +4178,7 @@ define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB88_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB118_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -3380,7 +4194,7 @@ define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB88_1;
+; SM90-NEXT: @%p1 bra $L__BB118_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -3412,7 +4226,7 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB89_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB119_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
@@ -3449,7 +4263,7 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB89_1;
+; SM90-NEXT: @%p1 bra $L__BB119_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -3458,16 +4272,34 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
-define <2 x i32> @max_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM90-LABEL: max_acq_rel_v2i32_global_cta(
+define <1 x i32> @max_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: max_acq_rel_v1i32_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b32 %r<3>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b32 %r1, [max_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @max_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: max_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
@@ -3525,6 +4357,23 @@ define <8 x i32> @max_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @max_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: max_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @max_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: max_acq_rel_v2i64_global_cta(
; SM90: {
@@ -3595,6 +4444,49 @@ define <8 x i64> @max_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: min_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.s8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB128_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r15, %r1;
+; SM90-NEXT: cvt.s8.s32 %rs2, %r8;
+; SM90-NEXT: min.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: and.b32 %r10, %r9, 255;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p1 bra $L__BB128_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: min_acq_rel_v2i8_global_cta(
; SM90: {
@@ -3620,7 +4512,7 @@ define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: cvt.s16.s8 %rs7, %rs6;
; SM90-NEXT: cvt.s16.s8 %rs8, %rs5;
; SM90-NEXT: mov.b32 %r12, {%rs8, %rs7};
-; SM90-NEXT: $L__BB96_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB129_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r19, %r1;
; SM90-NEXT: cvt.s8.s32 %rs3, %r9;
@@ -3639,7 +4531,7 @@ define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r19;
; SM90-NEXT: mov.b32 %r19, %r3;
-; SM90-NEXT: @%p1 bra $L__BB96_1;
+; SM90-NEXT: @%p1 bra $L__BB129_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r18, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -3665,7 +4557,7 @@ define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
-; SM90-NEXT: $L__BB97_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB130_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
; SM90-NEXT: setp.le.s32 %p1, %r4, %r3;
@@ -3693,7 +4585,7 @@ define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM90-NEXT: setp.ne.b32 %p5, %r1, %r26;
; SM90-NEXT: mov.b32 %r26, %r1;
-; SM90-NEXT: @%p5 bra $L__BB97_1;
+; SM90-NEXT: @%p5 bra $L__BB130_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -3719,7 +4611,7 @@ define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: cvt.u32.u64 %r49, %rd2;
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
-; SM90-NEXT: $L__BB98_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB131_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r49;
; SM90-NEXT: cvt.u64.u32 %rd4, %r50;
@@ -3809,7 +4701,7 @@ define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r50}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM90-NEXT: @%p9 bra $L__BB98_1;
+; SM90-NEXT: @%p9 bra $L__BB131_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
@@ -3818,6 +4710,48 @@ define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i16> @min_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: min_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB132_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: min.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB132_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: min_acq_rel_v2i16_global_cta(
; SM90: {
@@ -3830,13 +4764,13 @@ define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB99_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB133_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.s16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB99_1;
+; SM90-NEXT: @%p1 bra $L__BB133_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -3859,7 +4793,7 @@ define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB100_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB134_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -3875,7 +4809,7 @@ define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB100_1;
+; SM90-NEXT: @%p1 bra $L__BB134_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -3907,7 +4841,7 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB101_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB135_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
@@ -3944,7 +4878,7 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB101_1;
+; SM90-NEXT: @%p1 bra $L__BB135_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -3953,6 +4887,24 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
ret <8 x i16> %retval
}
+define <1 x i32> @min_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: min_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [min_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @min_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: min_acq_rel_v2i32_global_cta(
; SM90: {
@@ -4020,6 +4972,23 @@ define <8 x i32> @min_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i64> @min_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: min_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @min_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: min_acq_rel_v2i64_global_cta(
; SM90: {
@@ -4090,6 +5059,49 @@ define <8 x i64> @min_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
ret <8 x i64> %retval
}
+define <1 x i8> @umax_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: umax_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b8 %rs1, [umax_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB144_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: and.b16 %rs3, %rs2, 255;
+; SM90-NEXT: max.u16 %rs4, %rs3, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB144_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: umax_acq_rel_v2i8_global_cta(
; SM90: {
@@ -4111,7 +5123,7 @@ define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: shl.b32 %r8, %r7, %r1;
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: $L__BB108_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB145_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r18, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -4129,7 +5141,7 @@ define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM90-NEXT: mov.b32 %r18, %r3;
-; SM90-NEXT: @%p1 bra $L__BB108_1;
+; SM90-NEXT: @%p1 bra $L__BB145_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -4155,7 +5167,7 @@ define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
-; SM90-NEXT: $L__BB109_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB146_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
; SM90-NEXT: setp.gt.u32 %p1, %r4, %r3;
@@ -4175,7 +5187,7 @@ define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM90-NEXT: setp.ne.b32 %p5, %r1, %r18;
; SM90-NEXT: mov.b32 %r18, %r1;
-; SM90-NEXT: @%p5 bra $L__BB109_1;
+; SM90-NEXT: @%p5 bra $L__BB146_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -4203,7 +5215,7 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
-; SM90-NEXT: $L__BB110_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB147_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -4275,7 +5287,7 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM90-NEXT: @%p9 bra $L__BB110_1;
+; SM90-NEXT: @%p9 bra $L__BB147_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -4284,6 +5296,48 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
ret <8 x i8> %retval
}
+define <1 x i16> @umax_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: umax_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB148_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB148_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: umax_acq_rel_v2i16_global_cta(
; SM90: {
@@ -4296,13 +5350,13 @@ define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB111_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB149_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.u16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB111_1;
+; SM90-NEXT: @%p1 bra $L__BB149_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -4325,7 +5379,7 @@ define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB112_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB150_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -4341,7 +5395,7 @@ define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB112_1;
+; SM90-NEXT: @%p1 bra $L__BB150_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -4373,7 +5427,7 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB113_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB151_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
@@ -4410,7 +5464,7 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB113_1;
+; SM90-NEXT: @%p1 bra $L__BB151_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -4419,6 +5473,24 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
ret <8 x i16> %retval
}
+define <1 x i32> @umax_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: umax_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [umax_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @umax_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: umax_acq_rel_v2i32_global_cta(
; SM90: {
@@ -4486,6 +5558,23 @@ define <8 x i32> @umax_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i64> @umax_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: umax_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @umax_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: umax_acq_rel_v2i64_global_cta(
; SM90: {
@@ -4556,6 +5645,49 @@ define <8 x i64> @umax_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
ret <8 x i64> %retval
}
+define <1 x i8> @umin_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: umin_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b8 %rs1, [umin_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB160_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: and.b16 %rs3, %rs2, 255;
+; SM90-NEXT: min.u16 %rs4, %rs3, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB160_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: umin_acq_rel_v2i8_global_cta(
; SM90: {
@@ -4577,7 +5709,7 @@ define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: shl.b32 %r8, %r7, %r1;
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: $L__BB120_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB161_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r18, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -4595,7 +5727,7 @@ define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM90-NEXT: mov.b32 %r18, %r3;
-; SM90-NEXT: @%p1 bra $L__BB120_1;
+; SM90-NEXT: @%p1 bra $L__BB161_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -4621,7 +5753,7 @@ define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
-; SM90-NEXT: $L__BB121_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB162_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
; SM90-NEXT: setp.le.u32 %p1, %r4, %r3;
@@ -4641,7 +5773,7 @@ define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM90-NEXT: setp.ne.b32 %p5, %r1, %r18;
; SM90-NEXT: mov.b32 %r18, %r1;
-; SM90-NEXT: @%p5 bra $L__BB121_1;
+; SM90-NEXT: @%p5 bra $L__BB162_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -4669,7 +5801,7 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
-; SM90-NEXT: $L__BB122_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB163_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -4741,7 +5873,7 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM90-NEXT: @%p9 bra $L__BB122_1;
+; SM90-NEXT: @%p9 bra $L__BB163_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -4750,6 +5882,48 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
ret <8 x i8> %retval
}
+define <1 x i16> @umin_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: umin_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB164_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: min.u16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB164_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: umin_acq_rel_v2i16_global_cta(
; SM90: {
@@ -4762,13 +5936,13 @@ define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB123_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB165_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.u16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB123_1;
+; SM90-NEXT: @%p1 bra $L__BB165_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -4791,7 +5965,7 @@ define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB124_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB166_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -4807,7 +5981,7 @@ define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB124_1;
+; SM90-NEXT: @%p1 bra $L__BB166_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -4839,7 +6013,7 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB125_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB167_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
@@ -4876,7 +6050,7 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB125_1;
+; SM90-NEXT: @%p1 bra $L__BB167_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -4885,6 +6059,24 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
ret <8 x i16> %retval
}
+define <1 x i32> @umin_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: umin_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [umin_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @umin_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: umin_acq_rel_v2i32_global_cta(
; SM90: {
@@ -4952,6 +6144,23 @@ define <8 x i32> @umin_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i64> @umin_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: umin_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @umin_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: umin_acq_rel_v2i64_global_cta(
; SM90: {
@@ -5022,6 +6231,52 @@ define <8 x i64> @umin_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
ret <8 x i64> %retval
}
+define <1 x i8> @uinc_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b16 %rs<6>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b8 %rs1, [uinc_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB176_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r15, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: and.b16 %rs3, %rs2, 255;
+; SM90-NEXT: add.s16 %rs4, %rs2, 1;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: selp.b16 %rs5, 0, %rs4, %p1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM90-NEXT: and.b32 %r10, %r9, 255;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p2, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p2 bra $L__BB176_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: uinc_wrap_acq_rel_v2i8_global_cta(
; SM90: {
@@ -5044,7 +6299,7 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM90-NEXT: mov.b32 {%rs6, %rs7}, %r4;
-; SM90-NEXT: $L__BB132_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB177_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r18, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -5068,7 +6323,7 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM90-NEXT: setp.ne.b32 %p3, %r3, %r18;
; SM90-NEXT: mov.b32 %r18, %r3;
-; SM90-NEXT: @%p3 bra $L__BB132_1;
+; SM90-NEXT: @%p3 bra $L__BB177_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -5095,7 +6350,7 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
-; SM90-NEXT: $L__BB133_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB178_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r3, %r22, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
@@ -5127,7 +6382,7 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
; SM90-NEXT: setp.ne.b32 %p5, %r1, %r22;
; SM90-NEXT: mov.b32 %r22, %r1;
-; SM90-NEXT: @%p5 bra $L__BB133_1;
+; SM90-NEXT: @%p5 bra $L__BB178_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -5153,7 +6408,7 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
-; SM90-NEXT: $L__BB134_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB179_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -5227,7 +6482,7 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM90-NEXT: @%p9 bra $L__BB134_1;
+; SM90-NEXT: @%p9 bra $L__BB179_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -5236,6 +6491,50 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i8> %retval
}
+define <1 x i16> @uinc_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB180_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: add.s16 %rs3, %rs2, 1;
+; SM90-NEXT: setp.ge.u16 %p1, %rs2, %rs1;
+; SM90-NEXT: selp.b16 %rs4, 0, %rs3, %p1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p2, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p2 bra $L__BB180_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: uinc_wrap_acq_rel_v2i16_global_cta(
; SM90: {
@@ -5250,7 +6549,7 @@ define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r6, [%rd1];
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NEXT: $L__BB135_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB181_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b32 %r3, 65537;
; SM90-NEXT: add.s16x2 %r4, %r6, %r3;
@@ -5264,7 +6563,7 @@ define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r6, %r5;
; SM90-NEXT: setp.ne.b32 %p3, %r1, %r6;
; SM90-NEXT: mov.b32 %r6, %r1;
-; SM90-NEXT: @%p3 bra $L__BB135_1;
+; SM90-NEXT: @%p3 bra $L__BB181_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -5290,7 +6589,7 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: cvt.u32.u64 %r8, %rd2;
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; SM90-NEXT: $L__BB136_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB182_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r8;
; SM90-NEXT: cvt.u64.u32 %rd4, %r9;
@@ -5321,7 +6620,7 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r9}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r8, %rd11;
-; SM90-NEXT: @%p5 bra $L__BB136_1;
+; SM90-NEXT: @%p5 bra $L__BB182_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r8, %r9};
@@ -5356,7 +6655,7 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: cvt.u32.u64 %r14, %rd3;
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r3;
-; SM90-NEXT: $L__BB137_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB183_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r14;
; SM90-NEXT: cvt.u64.u32 %rd6, %r15;
@@ -5424,7 +6723,7 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: mov.b64 {_, %r15}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r14, %rd21;
; SM90-NEXT: setp.ne.b64 %p9, %rd25, 0;
-; SM90-NEXT: @%p9 bra $L__BB137_1;
+; SM90-NEXT: @%p9 bra $L__BB183_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r14, %r15, %r16, %r17};
@@ -5433,16 +6732,34 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i16> %retval
}
-define <2 x i32> @uinc_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM90-LABEL: uinc_wrap_acq_rel_v2i32_global_cta(
+define <1 x i32> @uinc_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v1i32_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b32 %r<3>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b32 %r1, [uinc_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @uinc_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
@@ -5500,6 +6817,34 @@ define <8 x i32> @uinc_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i32> %retval
}
+define <1 x i64> @uinc_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<7>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM90-NEXT: $L__BB188_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd4, %rd6, 1;
+; SM90-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM90-NEXT: selp.b64 %rd5, 0, %rd4, %p1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM90-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
+; SM90-NEXT: mov.b64 %rd6, %rd2;
+; SM90-NEXT: @%p2 bra $L__BB188_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: uinc_wrap_acq_rel_v2i64_global_cta(
; SM90: {
@@ -5518,7 +6863,7 @@ define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
; SM90-NEXT: mov.b128 {%rd14, %rd15}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB141_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB189_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s64 %rd7, %rd15, 1;
; SM90-NEXT: add.s64 %rd8, %rd14, 1;
@@ -5539,7 +6884,7 @@ define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: setp.ne.b64 %p3, %rd13, 0;
; SM90-NEXT: mov.b64 %rd14, %rd1;
; SM90-NEXT: mov.b64 %rd15, %rd2;
-; SM90-NEXT: @%p3 bra $L__BB141_1;
+; SM90-NEXT: @%p3 bra $L__BB189_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
@@ -5567,7 +6912,7 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
; SM90-NEXT: mov.b128 {%rd26, %rd27}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB142_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB190_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s64 %rd11, %rd27, 1;
; SM90-NEXT: add.s64 %rd12, %rd26, 1;
@@ -5588,7 +6933,7 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: setp.ne.b64 %p3, %rd17, 0;
; SM90-NEXT: mov.b64 %rd26, %rd5;
; SM90-NEXT: mov.b64 %rd27, %rd6;
-; SM90-NEXT: @%p3 bra $L__BB142_1;
+; SM90-NEXT: @%p3 bra $L__BB190_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
@@ -5598,7 +6943,7 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd28, %rd29}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB142_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB190_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s64 %rd19, %rd29, 1;
; SM90-NEXT: add.s64 %rd20, %rd28, 1;
@@ -5619,7 +6964,7 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: setp.ne.b64 %p6, %rd25, 0;
; SM90-NEXT: mov.b64 %rd28, %rd7;
; SM90-NEXT: mov.b64 %rd29, %rd8;
-; SM90-NEXT: @%p6 bra $L__BB142_3;
+; SM90-NEXT: @%p6 bra $L__BB190_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -5650,7 +6995,7 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB143_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB191_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd10, %rd51;
; SM90-NEXT: mov.b64 %rd9, %rd50;
@@ -5671,7 +7016,7 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: xor.b64 %rd24, %rd50, %rd9;
; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
; SM90-NEXT: setp.ne.b64 %p3, %rd25, 0;
-; SM90-NEXT: @%p3 bra $L__BB143_1;
+; SM90-NEXT: @%p3 bra $L__BB191_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd26, 0;
; SM90-NEXT: {
@@ -5681,7 +7026,7 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB143_3: // %atomicrmw.start6
+; SM90-NEXT: $L__BB191_3: // %atomicrmw.start6
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd12, %rd53;
; SM90-NEXT: mov.b64 %rd11, %rd52;
@@ -5702,7 +7047,7 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: xor.b64 %rd32, %rd52, %rd11;
; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
; SM90-NEXT: setp.ne.b64 %p6, %rd33, 0;
-; SM90-NEXT: @%p6 bra $L__BB143_3;
+; SM90-NEXT: @%p6 bra $L__BB191_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end5
; SM90-NEXT: mov.b64 %rd34, 0;
; SM90-NEXT: {
@@ -5712,7 +7057,7 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
; SM90-NEXT: mov.b128 {%rd54, %rd55}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB143_5: // %atomicrmw.start16
+; SM90-NEXT: $L__BB191_5: // %atomicrmw.start16
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s64 %rd35, %rd55, 1;
; SM90-NEXT: add.s64 %rd36, %rd54, 1;
@@ -5733,7 +7078,7 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: setp.ne.b64 %p9, %rd41, 0;
; SM90-NEXT: mov.b64 %rd54, %rd13;
; SM90-NEXT: mov.b64 %rd55, %rd14;
-; SM90-NEXT: @%p9 bra $L__BB143_5;
+; SM90-NEXT: @%p9 bra $L__BB191_5;
; SM90-NEXT: // %bb.6: // %atomicrmw.end15
; SM90-NEXT: mov.b64 %rd42, 0;
; SM90-NEXT: {
@@ -5743,7 +7088,7 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
; SM90-NEXT: mov.b128 {%rd56, %rd57}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB143_7: // %atomicrmw.start22
+; SM90-NEXT: $L__BB191_7: // %atomicrmw.start22
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s64 %rd43, %rd57, 1;
; SM90-NEXT: add.s64 %rd44, %rd56, 1;
@@ -5764,7 +7109,7 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: setp.ne.b64 %p12, %rd49, 0;
; SM90-NEXT: mov.b64 %rd56, %rd15;
; SM90-NEXT: mov.b64 %rd57, %rd16;
-; SM90-NEXT: @%p12 bra $L__BB143_7;
+; SM90-NEXT: @%p12 bra $L__BB191_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end21
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -5776,6 +7121,54 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i64> %retval
}
+define <1 x i8> @udec_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b16 %rs<7>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b8 %rs1, [udec_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB192_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r15, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: and.b16 %rs3, %rs2, 255;
+; SM90-NEXT: add.s16 %rs4, %rs2, -1;
+; SM90-NEXT: setp.eq.b16 %p1, %rs3, 0;
+; SM90-NEXT: setp.gt.u16 %p2, %rs3, %rs1;
+; SM90-NEXT: selp.b16 %rs5, %rs1, %rs4, %p2;
+; SM90-NEXT: selp.b16 %rs6, %rs1, %rs5, %p1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM90-NEXT: and.b32 %r10, %r9, 255;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p3 bra $L__BB192_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: udec_wrap_acq_rel_v2i8_global_cta(
; SM90: {
@@ -5798,7 +7191,7 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM90-NEXT: mov.b32 {%rs8, %rs9}, %r4;
-; SM90-NEXT: $L__BB144_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB193_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r15, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -5824,7 +7217,7 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM90-NEXT: setp.ne.b32 %p5, %r3, %r15;
; SM90-NEXT: mov.b32 %r15, %r3;
-; SM90-NEXT: @%p5 bra $L__BB144_1;
+; SM90-NEXT: @%p5 bra $L__BB193_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -5851,7 +7244,7 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
-; SM90-NEXT: $L__BB145_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB194_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r3, %r26, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
@@ -5891,7 +7284,7 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM90-NEXT: setp.ne.b32 %p9, %r1, %r26;
; SM90-NEXT: mov.b32 %r26, %r1;
-; SM90-NEXT: @%p9 bra $L__BB145_1;
+; SM90-NEXT: @%p9 bra $L__BB194_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -5919,7 +7312,7 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs17, %r11;
; SM90-NEXT: cvt.u16.u32 %rs18, %r12;
-; SM90-NEXT: $L__BB146_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB195_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -6015,7 +7408,7 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: setp.ne.b64 %p17, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM90-NEXT: @%p17 bra $L__BB146_1;
+; SM90-NEXT: @%p17 bra $L__BB195_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -6024,6 +7417,52 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i8> %retval
}
+define <1 x i16> @udec_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b16 %rs<6>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB196_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: add.s16 %rs3, %rs2, -1;
+; SM90-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM90-NEXT: setp.gt.u16 %p2, %rs2, %rs1;
+; SM90-NEXT: selp.b16 %rs4, %rs1, %rs3, %p2;
+; SM90-NEXT: selp.b16 %rs5, %rs1, %rs4, %p1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p3, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p3 bra $L__BB196_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: udec_wrap_acq_rel_v2i16_global_cta(
; SM90: {
@@ -6038,7 +7477,7 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM90-NEXT: $L__BB147_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB197_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; SM90-NEXT: add.s16 %rs3, %rs1, -1;
@@ -6055,7 +7494,7 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p5, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p5 bra $L__BB147_1;
+; SM90-NEXT: @%p5 bra $L__BB197_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -6081,7 +7520,7 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r2;
; SM90-NEXT: mov.b32 {%rs11, %rs12}, %r1;
-; SM90-NEXT: $L__BB148_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB198_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -6119,7 +7558,7 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p9 bra $L__BB148_1;
+; SM90-NEXT: @%p9 bra $L__BB198_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -6154,7 +7593,7 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
; SM90-NEXT: mov.b32 {%rs17, %rs18}, %r4;
; SM90-NEXT: mov.b32 {%rs19, %rs20}, %r3;
-; SM90-NEXT: $L__BB149_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB199_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
@@ -6237,7 +7676,7 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p17, %rd25, 0;
-; SM90-NEXT: @%p17 bra $L__BB149_1;
+; SM90-NEXT: @%p17 bra $L__BB199_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -6246,6 +7685,24 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i16> %retval
}
+define <1 x i32> @udec_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [udec_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @udec_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: udec_wrap_acq_rel_v2i32_global_cta(
; SM90: {
@@ -6313,6 +7770,36 @@ define <8 x i32> @udec_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i32> %retval
}
+define <1 x i64> @udec_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b64 %rd<8>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd7, [%rd3];
+; SM90-NEXT: $L__BB204_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd4, %rd7, -1;
+; SM90-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM90-NEXT: setp.gt.u64 %p2, %rd7, %rd1;
+; SM90-NEXT: selp.b64 %rd5, %rd1, %rd4, %p2;
+; SM90-NEXT: selp.b64 %rd6, %rd1, %rd5, %p1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd7, %rd6;
+; SM90-NEXT: setp.ne.b64 %p3, %rd2, %rd7;
+; SM90-NEXT: mov.b64 %rd7, %rd2;
+; SM90-NEXT: @%p3 bra $L__BB204_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: udec_wrap_acq_rel_v2i64_global_cta(
; SM90: {
@@ -6331,7 +7818,7 @@ define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
; SM90-NEXT: mov.b128 {%rd16, %rd17}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB153_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB205_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s64 %rd7, %rd17, -1;
; SM90-NEXT: add.s64 %rd8, %rd16, -1;
@@ -6356,7 +7843,7 @@ define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: setp.ne.b64 %p5, %rd15, 0;
; SM90-NEXT: mov.b64 %rd16, %rd1;
; SM90-NEXT: mov.b64 %rd17, %rd2;
-; SM90-NEXT: @%p5 bra $L__BB153_1;
+; SM90-NEXT: @%p5 bra $L__BB205_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
@@ -6384,7 +7871,7 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
; SM90-NEXT: mov.b128 {%rd30, %rd31}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB154_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB206_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s64 %rd11, %rd31, -1;
; SM90-NEXT: add.s64 %rd12, %rd30, -1;
@@ -6409,7 +7896,7 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: setp.ne.b64 %p5, %rd19, 0;
; SM90-NEXT: mov.b64 %rd30, %rd5;
; SM90-NEXT: mov.b64 %rd31, %rd6;
-; SM90-NEXT: @%p5 bra $L__BB154_1;
+; SM90-NEXT: @%p5 bra $L__BB206_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd20, 0;
; SM90-NEXT: {
@@ -6419,7 +7906,7 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd32, %rd33}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB154_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB206_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s64 %rd21, %rd33, -1;
; SM90-NEXT: add.s64 %rd22, %rd32, -1;
@@ -6444,7 +7931,7 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: setp.ne.b64 %p10, %rd29, 0;
; SM90-NEXT: mov.b64 %rd32, %rd7;
; SM90-NEXT: mov.b64 %rd33, %rd8;
-; SM90-NEXT: @%p10 bra $L__BB154_3;
+; SM90-NEXT: @%p10 bra $L__BB206_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -6475,7 +7962,7 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
; SM90-NEXT: mov.b128 {%rd58, %rd59}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB155_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB207_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd10, %rd59;
; SM90-NEXT: mov.b64 %rd9, %rd58;
@@ -6500,7 +7987,7 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: xor.b64 %rd26, %rd58, %rd9;
; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
; SM90-NEXT: setp.ne.b64 %p5, %rd27, 0;
-; SM90-NEXT: @%p5 bra $L__BB155_1;
+; SM90-NEXT: @%p5 bra $L__BB207_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd28, 0;
; SM90-NEXT: {
@@ -6510,7 +7997,7 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd60, %rd61}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB155_3: // %atomicrmw.start6
+; SM90-NEXT: $L__BB207_3: // %atomicrmw.start6
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd12, %rd61;
; SM90-NEXT: mov.b64 %rd11, %rd60;
@@ -6535,7 +8022,7 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: xor.b64 %rd36, %rd60, %rd11;
; SM90-NEXT: or.b64 %rd37, %rd36, %rd35;
; SM90-NEXT: setp.ne.b64 %p10, %rd37, 0;
-; SM90-NEXT: @%p10 bra $L__BB155_3;
+; SM90-NEXT: @%p10 bra $L__BB207_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end5
; SM90-NEXT: mov.b64 %rd38, 0;
; SM90-NEXT: {
@@ -6545,7 +8032,7 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
; SM90-NEXT: mov.b128 {%rd62, %rd63}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB155_5: // %atomicrmw.start16
+; SM90-NEXT: $L__BB207_5: // %atomicrmw.start16
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s64 %rd39, %rd63, -1;
; SM90-NEXT: add.s64 %rd40, %rd62, -1;
@@ -6570,7 +8057,7 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: setp.ne.b64 %p15, %rd47, 0;
; SM90-NEXT: mov.b64 %rd62, %rd13;
; SM90-NEXT: mov.b64 %rd63, %rd14;
-; SM90-NEXT: @%p15 bra $L__BB155_5;
+; SM90-NEXT: @%p15 bra $L__BB207_5;
; SM90-NEXT: // %bb.6: // %atomicrmw.end15
; SM90-NEXT: mov.b64 %rd48, 0;
; SM90-NEXT: {
@@ -6580,7 +8067,7 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
; SM90-NEXT: mov.b128 {%rd64, %rd65}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB155_7: // %atomicrmw.start22
+; SM90-NEXT: $L__BB207_7: // %atomicrmw.start22
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: add.s64 %rd49, %rd65, -1;
; SM90-NEXT: add.s64 %rd50, %rd64, -1;
@@ -6605,7 +8092,7 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: setp.ne.b64 %p20, %rd57, 0;
; SM90-NEXT: mov.b64 %rd64, %rd15;
; SM90-NEXT: mov.b64 %rd65, %rd16;
-; SM90-NEXT: @%p20 bra $L__BB155_7;
+; SM90-NEXT: @%p20 bra $L__BB207_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end21
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -6617,6 +8104,52 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i64> %retval
}
+define <1 x i8> @usub_cond_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b16 %rs<6>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b8 %rs1, [usub_cond_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB208_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r15, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: and.b16 %rs3, %rs2, 255;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: sub.s16 %rs4, %rs2, %rs1;
+; SM90-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM90-NEXT: and.b32 %r10, %r9, 255;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p2, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p2 bra $L__BB208_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: usub_cond_acq_rel_v2i8_global_cta(
; SM90: {
@@ -6639,7 +8172,7 @@ define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM90-NEXT: mov.b32 {%rs6, %rs7}, %r4;
-; SM90-NEXT: $L__BB156_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB209_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r15, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -6661,7 +8194,7 @@ define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM90-NEXT: setp.ne.b32 %p3, %r3, %r15;
; SM90-NEXT: mov.b32 %r15, %r3;
-; SM90-NEXT: @%p3 bra $L__BB156_1;
+; SM90-NEXT: @%p3 bra $L__BB209_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -6692,7 +8225,7 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: cvt.u16.u32 %rs2, %r7;
; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
; SM90-NEXT: cvt.u16.u32 %rs4, %r3;
-; SM90-NEXT: $L__BB157_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB210_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r22, 0, 0x7770U;
; SM90-NEXT: setp.ge.u32 %p1, %r4, %r3;
@@ -6724,7 +8257,7 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
; SM90-NEXT: setp.ne.b32 %p5, %r1, %r22;
; SM90-NEXT: mov.b32 %r22, %r1;
-; SM90-NEXT: @%p5 bra $L__BB157_1;
+; SM90-NEXT: @%p5 bra $L__BB210_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -6752,7 +8285,7 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs13, %r5;
; SM90-NEXT: cvt.u16.u32 %rs15, %r3;
-; SM90-NEXT: $L__BB158_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB211_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
@@ -6832,7 +8365,7 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM90-NEXT: @%p9 bra $L__BB158_1;
+; SM90-NEXT: @%p9 bra $L__BB211_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -6841,6 +8374,50 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i8> %retval
}
+define <1 x i16> @usub_cond_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB212_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: setp.ge.u16 %p1, %rs2, %rs1;
+; SM90-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: selp.b16 %rs4, %rs3, %rs2, %p1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p2, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p2 bra $L__BB212_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: usub_cond_acq_rel_v2i16_global_cta(
; SM90: {
@@ -6855,7 +8432,7 @@ define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NEXT: $L__BB159_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB213_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r4;
; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
@@ -6868,7 +8445,7 @@ define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p3, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p3 bra $L__BB159_1;
+; SM90-NEXT: @%p3 bra $L__BB213_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -6894,7 +8471,7 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; SM90-NEXT: $L__BB160_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB214_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -6924,7 +8501,7 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p5 bra $L__BB160_1;
+; SM90-NEXT: @%p5 bra $L__BB214_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -6959,7 +8536,7 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r3;
-; SM90-NEXT: $L__BB161_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB215_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
@@ -7026,7 +8603,7 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p9, %rd25, 0;
-; SM90-NEXT: @%p9 bra $L__BB161_1;
+; SM90-NEXT: @%p9 bra $L__BB215_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -7035,6 +8612,35 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i16> %retval
}
+define <1 x i32> @usub_cond_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [usub_cond_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB216_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u32 %p1, %r5, %r1;
+; SM90-NEXT: sub.s32 %r3, %r5, %r1;
+; SM90-NEXT: selp.b32 %r4, %r3, %r5, %p1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p2, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p2 bra $L__BB216_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: usub_cond_acq_rel_v2i32_global_cta(
; SM90: {
@@ -7049,7 +8655,7 @@ define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: $L__BB162_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB217_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -7069,7 +8675,7 @@ define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM90-NEXT: @%p3 bra $L__BB162_1;
+; SM90-NEXT: @%p3 bra $L__BB217_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -7101,7 +8707,7 @@ define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: cvt.u32.u64 %r15, %rd4;
; SM90-NEXT: mov.b64 {_, %r14}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r13, %rd3;
-; SM90-NEXT: $L__BB163_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB218_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r13;
; SM90-NEXT: cvt.u64.u32 %rd6, %r14;
@@ -7146,7 +8752,7 @@ define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: mov.b64 {_, %r14}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r13, %rd21;
; SM90-NEXT: setp.ne.b64 %p5, %rd25, 0;
-; SM90-NEXT: @%p5 bra $L__BB163_1;
+; SM90-NEXT: @%p5 bra $L__BB218_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
@@ -7179,7 +8785,7 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: cvt.u32.u64 %r27, %rd4;
; SM90-NEXT: mov.b64 {_, %r26}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r25, %rd3;
-; SM90-NEXT: $L__BB164_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB219_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r25;
; SM90-NEXT: cvt.u64.u32 %rd6, %r26;
@@ -7224,7 +8830,7 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: mov.b64 {_, %r26}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r25, %rd21;
; SM90-NEXT: setp.ne.b64 %p5, %rd25, 0;
-; SM90-NEXT: @%p5 bra $L__BB164_1;
+; SM90-NEXT: @%p5 bra $L__BB219_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd26, 0;
; SM90-NEXT: {
@@ -7238,7 +8844,7 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: cvt.u32.u64 %r31, %rd28;
; SM90-NEXT: mov.b64 {_, %r30}, %rd27;
; SM90-NEXT: cvt.u32.u64 %r29, %rd27;
-; SM90-NEXT: $L__BB164_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB219_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd29, %r29;
; SM90-NEXT: cvt.u64.u32 %rd30, %r30;
@@ -7283,7 +8889,7 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: mov.b64 {_, %r30}, %rd45;
; SM90-NEXT: cvt.u32.u64 %r29, %rd45;
; SM90-NEXT: setp.ne.b64 %p10, %rd49, 0;
-; SM90-NEXT: @%p10 bra $L__BB164_3;
+; SM90-NEXT: @%p10 bra $L__BB219_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
@@ -7293,6 +8899,34 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i32> %retval
}
+define <1 x i64> @usub_cond_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<7>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM90-NEXT: $L__BB220_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM90-NEXT: sub.s64 %rd4, %rd6, %rd1;
+; SM90-NEXT: selp.b64 %rd5, %rd4, %rd6, %p1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM90-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
+; SM90-NEXT: mov.b64 %rd6, %rd2;
+; SM90-NEXT: @%p2 bra $L__BB220_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: usub_cond_acq_rel_v2i64_global_cta(
; SM90: {
@@ -7311,7 +8945,7 @@ define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
; SM90-NEXT: mov.b128 {%rd14, %rd15}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB165_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB221_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.ge.u64 %p1, %rd15, %rd5;
; SM90-NEXT: setp.ge.u64 %p2, %rd14, %rd4;
@@ -7332,7 +8966,7 @@ define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: setp.ne.b64 %p3, %rd13, 0;
; SM90-NEXT: mov.b64 %rd14, %rd1;
; SM90-NEXT: mov.b64 %rd15, %rd2;
-; SM90-NEXT: @%p3 bra $L__BB165_1;
+; SM90-NEXT: @%p3 bra $L__BB221_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
@@ -7360,7 +8994,7 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
; SM90-NEXT: mov.b128 {%rd26, %rd27}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB166_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB222_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.ge.u64 %p1, %rd27, %rd2;
; SM90-NEXT: setp.ge.u64 %p2, %rd26, %rd1;
@@ -7381,7 +9015,7 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: setp.ne.b64 %p3, %rd17, 0;
; SM90-NEXT: mov.b64 %rd26, %rd5;
; SM90-NEXT: mov.b64 %rd27, %rd6;
-; SM90-NEXT: @%p3 bra $L__BB166_1;
+; SM90-NEXT: @%p3 bra $L__BB222_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
@@ -7391,7 +9025,7 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd28, %rd29}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB166_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB222_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.ge.u64 %p4, %rd29, %rd4;
; SM90-NEXT: setp.ge.u64 %p5, %rd28, %rd3;
@@ -7412,7 +9046,7 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: setp.ne.b64 %p6, %rd25, 0;
; SM90-NEXT: mov.b64 %rd28, %rd7;
; SM90-NEXT: mov.b64 %rd29, %rd8;
-; SM90-NEXT: @%p6 bra $L__BB166_3;
+; SM90-NEXT: @%p6 bra $L__BB222_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -7443,7 +9077,7 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB167_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB223_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd10, %rd51;
; SM90-NEXT: mov.b64 %rd9, %rd50;
@@ -7464,7 +9098,7 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: xor.b64 %rd24, %rd50, %rd9;
; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
; SM90-NEXT: setp.ne.b64 %p3, %rd25, 0;
-; SM90-NEXT: @%p3 bra $L__BB167_1;
+; SM90-NEXT: @%p3 bra $L__BB223_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd26, 0;
; SM90-NEXT: {
@@ -7474,7 +9108,7 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB167_3: // %atomicrmw.start6
+; SM90-NEXT: $L__BB223_3: // %atomicrmw.start6
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd12, %rd53;
; SM90-NEXT: mov.b64 %rd11, %rd52;
@@ -7495,7 +9129,7 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: xor.b64 %rd32, %rd52, %rd11;
; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
; SM90-NEXT: setp.ne.b64 %p6, %rd33, 0;
-; SM90-NEXT: @%p6 bra $L__BB167_3;
+; SM90-NEXT: @%p6 bra $L__BB223_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end5
; SM90-NEXT: mov.b64 %rd34, 0;
; SM90-NEXT: {
@@ -7505,7 +9139,7 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
; SM90-NEXT: mov.b128 {%rd54, %rd55}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB167_5: // %atomicrmw.start16
+; SM90-NEXT: $L__BB223_5: // %atomicrmw.start16
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.ge.u64 %p7, %rd55, %rd2;
; SM90-NEXT: setp.ge.u64 %p8, %rd54, %rd1;
@@ -7526,7 +9160,7 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: setp.ne.b64 %p9, %rd41, 0;
; SM90-NEXT: mov.b64 %rd54, %rd13;
; SM90-NEXT: mov.b64 %rd55, %rd14;
-; SM90-NEXT: @%p9 bra $L__BB167_5;
+; SM90-NEXT: @%p9 bra $L__BB223_5;
; SM90-NEXT: // %bb.6: // %atomicrmw.end15
; SM90-NEXT: mov.b64 %rd42, 0;
; SM90-NEXT: {
@@ -7536,7 +9170,7 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
; SM90-NEXT: mov.b128 {%rd56, %rd57}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB167_7: // %atomicrmw.start22
+; SM90-NEXT: $L__BB223_7: // %atomicrmw.start22
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.ge.u64 %p10, %rd57, %rd4;
; SM90-NEXT: setp.ge.u64 %p11, %rd56, %rd3;
@@ -7557,7 +9191,7 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: setp.ne.b64 %p12, %rd49, 0;
; SM90-NEXT: mov.b64 %rd56, %rd15;
; SM90-NEXT: mov.b64 %rd57, %rd16;
-; SM90-NEXT: @%p12 bra $L__BB167_7;
+; SM90-NEXT: @%p12 bra $L__BB223_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end21
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -7569,6 +9203,50 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
ret <8 x i64> %retval
}
+define <1 x i8> @usub_sat_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<6>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b8 %rs1, [usub_sat_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB224_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: and.b16 %rs3, %rs2, 255;
+; SM90-NEXT: max.u16 %rs4, %rs3, %rs1;
+; SM90-NEXT: sub.s16 %rs5, %rs4, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB224_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: usub_sat_acq_rel_v2i8_global_cta(
; SM90: {
@@ -7591,7 +9269,7 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r4;
-; SM90-NEXT: $L__BB168_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB225_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r18, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -7611,7 +9289,7 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM90-NEXT: mov.b32 %r18, %r3;
-; SM90-NEXT: @%p1 bra $L__BB168_1;
+; SM90-NEXT: @%p1 bra $L__BB225_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -7642,7 +9320,7 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
; SM90-NEXT: cvt.u16.u32 %rs9, %r10;
; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs13, %r13;
-; SM90-NEXT: $L__BB169_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB226_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -7670,7 +9348,7 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM90-NEXT: mov.b32 %r18, %r1;
-; SM90-NEXT: @%p1 bra $L__BB169_1;
+; SM90-NEXT: @%p1 bra $L__BB226_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -7704,7 +9382,7 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: cvt.u16.u32 %rs13, %r13;
; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM90-NEXT: $L__BB170_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB227_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -7770,7 +9448,7 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB170_1;
+; SM90-NEXT: @%p1 bra $L__BB227_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -7779,6 +9457,49 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
ret <8 x i8> %retval
}
+define <1 x i16> @usub_sat_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB228_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM90-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB228_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM90-LABEL: usub_sat_acq_rel_v2i16_global_cta(
; SM90: {
@@ -7793,7 +9514,7 @@ define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; SM90-NEXT: $L__BB171_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB229_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.u16x2 %r3, %r5, %r2;
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r3;
@@ -7803,7 +9524,7 @@ define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM90-NEXT: mov.b32 %r5, %r1;
-; SM90-NEXT: @%p1 bra $L__BB171_1;
+; SM90-NEXT: @%p1 bra $L__BB229_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -7829,7 +9550,7 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r2;
-; SM90-NEXT: $L__BB172_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB230_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.u16x2 %r3, %r7, %r1;
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r3;
@@ -7853,7 +9574,7 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB172_1;
+; SM90-NEXT: @%p1 bra $L__BB230_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -7888,7 +9609,7 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: cvt.u32.u64 %r13, %rd3;
; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r2;
-; SM90-NEXT: $L__BB173_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB231_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.u16x2 %r5, %r13, %r1;
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r5;
@@ -7943,7 +9664,7 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: mov.b64 {_, %r14}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r13, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB173_1;
+; SM90-NEXT: @%p1 bra $L__BB231_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
@@ -7952,6 +9673,34 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i16> %retval
}
+define <1 x i32> @usub_sat_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [usub_sat_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB232_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u32 %r3, %r5, %r1;
+; SM90-NEXT: sub.s32 %r4, %r3, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p1 bra $L__BB232_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: usub_sat_acq_rel_v2i32_global_cta(
; SM90: {
@@ -7966,7 +9715,7 @@ define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: $L__BB174_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB233_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.u32 %r3, %r7, %r1;
; SM90-NEXT: sub.s32 %r4, %r3, %r1;
@@ -7984,7 +9733,7 @@ define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB174_1;
+; SM90-NEXT: @%p1 bra $L__BB233_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -8016,7 +9765,7 @@ define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: cvt.u32.u64 %r15, %rd4;
; SM90-NEXT: mov.b64 {_, %r14}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r13, %rd3;
-; SM90-NEXT: $L__BB175_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB234_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.u32 %r5, %r13, %r1;
; SM90-NEXT: sub.s32 %r6, %r5, %r1;
@@ -8057,7 +9806,7 @@ define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: mov.b64 {_, %r14}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r13, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB175_1;
+; SM90-NEXT: @%p1 bra $L__BB234_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
@@ -8090,7 +9839,7 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: cvt.u32.u64 %r27, %rd4;
; SM90-NEXT: mov.b64 {_, %r26}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r25, %rd3;
-; SM90-NEXT: $L__BB176_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB235_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.u32 %r9, %r25, %r1;
; SM90-NEXT: sub.s32 %r10, %r9, %r1;
@@ -8131,7 +9880,7 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: mov.b64 {_, %r26}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r25, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB176_1;
+; SM90-NEXT: @%p1 bra $L__BB235_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd26, 0;
; SM90-NEXT: {
@@ -8145,7 +9894,7 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: cvt.u32.u64 %r31, %rd28;
; SM90-NEXT: mov.b64 {_, %r30}, %rd27;
; SM90-NEXT: cvt.u32.u64 %r29, %rd27;
-; SM90-NEXT: $L__BB176_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB235_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.u32 %r17, %r29, %r5;
; SM90-NEXT: sub.s32 %r18, %r17, %r5;
@@ -8186,7 +9935,7 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: mov.b64 {_, %r30}, %rd45;
; SM90-NEXT: cvt.u32.u64 %r29, %rd45;
; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
-; SM90-NEXT: @%p2 bra $L__BB176_3;
+; SM90-NEXT: @%p2 bra $L__BB235_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
@@ -8196,6 +9945,33 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i32> %retval
}
+define <1 x i64> @usub_sat_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<7>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM90-NEXT: $L__BB236_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u64 %rd4, %rd6, %rd1;
+; SM90-NEXT: sub.s64 %rd5, %rd4, %rd1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
+; SM90-NEXT: mov.b64 %rd6, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB236_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM90-LABEL: usub_sat_acq_rel_v2i64_global_cta(
; SM90: {
@@ -8214,7 +9990,7 @@ define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
; SM90-NEXT: mov.b128 {%rd14, %rd15}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB177_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB237_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.u64 %rd7, %rd14, %rd4;
; SM90-NEXT: sub.s64 %rd8, %rd7, %rd4;
@@ -8233,7 +10009,7 @@ define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: setp.ne.b64 %p1, %rd13, 0;
; SM90-NEXT: mov.b64 %rd14, %rd1;
; SM90-NEXT: mov.b64 %rd15, %rd2;
-; SM90-NEXT: @%p1 bra $L__BB177_1;
+; SM90-NEXT: @%p1 bra $L__BB237_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
@@ -8261,7 +10037,7 @@ define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
; SM90-NEXT: mov.b128 {%rd26, %rd27}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB178_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB238_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.u64 %rd11, %rd26, %rd1;
; SM90-NEXT: sub.s64 %rd12, %rd11, %rd1;
@@ -8280,7 +10056,7 @@ define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.ne.b64 %p1, %rd17, 0;
; SM90-NEXT: mov.b64 %rd26, %rd5;
; SM90-NEXT: mov.b64 %rd27, %rd6;
-; SM90-NEXT: @%p1 bra $L__BB178_1;
+; SM90-NEXT: @%p1 bra $L__BB238_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
@@ -8290,7 +10066,7 @@ define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd28, %rd29}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB178_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB238_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.u64 %rd19, %rd28, %rd3;
; SM90-NEXT: sub.s64 %rd20, %rd19, %rd3;
@@ -8309,7 +10085,7 @@ define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.ne.b64 %p2, %rd25, 0;
; SM90-NEXT: mov.b64 %rd28, %rd7;
; SM90-NEXT: mov.b64 %rd29, %rd8;
-; SM90-NEXT: @%p2 bra $L__BB178_3;
+; SM90-NEXT: @%p2 bra $L__BB238_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -8340,7 +10116,7 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB179_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB239_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd10, %rd51;
; SM90-NEXT: mov.b64 %rd9, %rd50;
@@ -8359,7 +10135,7 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: xor.b64 %rd24, %rd50, %rd9;
; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB179_1;
+; SM90-NEXT: @%p1 bra $L__BB239_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd26, 0;
; SM90-NEXT: {
@@ -8369,7 +10145,7 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB179_3: // %atomicrmw.start6
+; SM90-NEXT: $L__BB239_3: // %atomicrmw.start6
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd12, %rd53;
; SM90-NEXT: mov.b64 %rd11, %rd52;
@@ -8388,7 +10164,7 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: xor.b64 %rd32, %rd52, %rd11;
; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
; SM90-NEXT: setp.ne.b64 %p2, %rd33, 0;
-; SM90-NEXT: @%p2 bra $L__BB179_3;
+; SM90-NEXT: @%p2 bra $L__BB239_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end5
; SM90-NEXT: mov.b64 %rd34, 0;
; SM90-NEXT: {
@@ -8398,7 +10174,7 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
; SM90-NEXT: mov.b128 {%rd54, %rd55}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB179_5: // %atomicrmw.start16
+; SM90-NEXT: $L__BB239_5: // %atomicrmw.start16
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.u64 %rd35, %rd54, %rd1;
; SM90-NEXT: sub.s64 %rd36, %rd35, %rd1;
@@ -8417,7 +10193,7 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: setp.ne.b64 %p3, %rd41, 0;
; SM90-NEXT: mov.b64 %rd54, %rd13;
; SM90-NEXT: mov.b64 %rd55, %rd14;
-; SM90-NEXT: @%p3 bra $L__BB179_5;
+; SM90-NEXT: @%p3 bra $L__BB239_5;
; SM90-NEXT: // %bb.6: // %atomicrmw.end15
; SM90-NEXT: mov.b64 %rd42, 0;
; SM90-NEXT: {
@@ -8427,7 +10203,7 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
; SM90-NEXT: mov.b128 {%rd56, %rd57}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB179_7: // %atomicrmw.start22
+; SM90-NEXT: $L__BB239_7: // %atomicrmw.start22
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.u64 %rd43, %rd56, %rd3;
; SM90-NEXT: sub.s64 %rd44, %rd43, %rd3;
@@ -8446,7 +10222,7 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: setp.ne.b64 %p4, %rd49, 0;
; SM90-NEXT: mov.b64 %rd56, %rd15;
; SM90-NEXT: mov.b64 %rd57, %rd16;
-; SM90-NEXT: @%p4 bra $L__BB179_7;
+; SM90-NEXT: @%p4 bra $L__BB239_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end21
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -8458,6 +10234,24 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i64> %retval
}
+define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM90-LABEL: fadd_acq_rel_v1f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM90-LABEL: fadd_acq_rel_v2f32_global_cta(
; SM90: {
@@ -8525,6 +10319,33 @@ define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
ret <8 x float> %retval
}
+define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM90-LABEL: fsub_acq_rel_v1f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB244_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f32 %r3, %r4, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM90-NEXT: mov.b32 %r4, %r2;
+; SM90-NEXT: @%p1 bra $L__BB244_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM90-LABEL: fsub_acq_rel_v2f32_global_cta(
; SM90: {
@@ -8539,7 +10360,7 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB183_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB245_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.f32 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -8555,7 +10376,7 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB183_1;
+; SM90-NEXT: @%p1 bra $L__BB245_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -8587,7 +10408,7 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB184_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB246_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.f32 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -8624,7 +10445,7 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB184_1;
+; SM90-NEXT: @%p1 bra $L__BB246_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -8657,7 +10478,7 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB185_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB247_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.f32 %r9, %r17, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
@@ -8694,7 +10515,7 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB185_1;
+; SM90-NEXT: @%p1 bra $L__BB247_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd26, 0;
; SM90-NEXT: {
@@ -8708,7 +10529,7 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
-; SM90-NEXT: $L__BB185_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB247_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.f32 %r13, %r21, %r5;
; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
@@ -8745,7 +10566,7 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
-; SM90-NEXT: @%p2 bra $L__BB185_3;
+; SM90-NEXT: @%p2 bra $L__BB247_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -8755,6 +10576,33 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
ret <8 x float> %retval
}
+define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM90-LABEL: fmin_acq_rel_v1f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB248_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f32 %r3, %r4, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM90-NEXT: mov.b32 %r4, %r2;
+; SM90-NEXT: @%p1 bra $L__BB248_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM90-LABEL: fmin_acq_rel_v2f32_global_cta(
; SM90: {
@@ -8769,7 +10617,7 @@ define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB186_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB249_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.f32 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -8785,7 +10633,7 @@ define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB186_1;
+; SM90-NEXT: @%p1 bra $L__BB249_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -8817,7 +10665,7 @@ define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB187_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB250_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.f32 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -8854,7 +10702,7 @@ define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB187_1;
+; SM90-NEXT: @%p1 bra $L__BB250_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -8887,7 +10735,7 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB188_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB251_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.f32 %r9, %r17, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
@@ -8924,7 +10772,7 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB188_1;
+; SM90-NEXT: @%p1 bra $L__BB251_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd26, 0;
; SM90-NEXT: {
@@ -8938,7 +10786,7 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
-; SM90-NEXT: $L__BB188_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB251_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.f32 %r13, %r21, %r5;
; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
@@ -8975,7 +10823,7 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
-; SM90-NEXT: @%p2 bra $L__BB188_3;
+; SM90-NEXT: @%p2 bra $L__BB251_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -8985,6 +10833,33 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
ret <8 x float> %retval
}
+define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM90-LABEL: fmax_acq_rel_v1f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB252_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f32 %r3, %r4, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM90-NEXT: mov.b32 %r4, %r2;
+; SM90-NEXT: @%p1 bra $L__BB252_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM90-LABEL: fmax_acq_rel_v2f32_global_cta(
; SM90: {
@@ -8999,7 +10874,7 @@ define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB189_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB253_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.f32 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -9015,7 +10890,7 @@ define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB189_1;
+; SM90-NEXT: @%p1 bra $L__BB253_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -9047,7 +10922,7 @@ define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB190_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB254_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.f32 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -9084,7 +10959,7 @@ define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB190_1;
+; SM90-NEXT: @%p1 bra $L__BB254_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -9117,7 +10992,7 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB191_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB255_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.f32 %r9, %r17, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
@@ -9154,7 +11029,7 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB191_1;
+; SM90-NEXT: @%p1 bra $L__BB255_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd26, 0;
; SM90-NEXT: {
@@ -9168,7 +11043,7 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
-; SM90-NEXT: $L__BB191_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB255_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.f32 %r13, %r21, %r5;
; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
@@ -9205,7 +11080,7 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
-; SM90-NEXT: @%p2 bra $L__BB191_3;
+; SM90-NEXT: @%p2 bra $L__BB255_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -9215,6 +11090,33 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
ret <8 x float> %retval
}
+define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM90-LABEL: fminimum_acq_rel_v1f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB256_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f32 %r3, %r4, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM90-NEXT: mov.b32 %r4, %r2;
+; SM90-NEXT: @%p1 bra $L__BB256_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM90-LABEL: fminimum_acq_rel_v2f32_global_cta(
; SM90: {
@@ -9229,7 +11131,7 @@ define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB192_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB257_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.NaN.f32 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -9245,7 +11147,7 @@ define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB192_1;
+; SM90-NEXT: @%p1 bra $L__BB257_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -9277,7 +11179,7 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB193_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB258_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.NaN.f32 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -9314,7 +11216,7 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB193_1;
+; SM90-NEXT: @%p1 bra $L__BB258_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -9347,7 +11249,7 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB194_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB259_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.NaN.f32 %r9, %r17, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
@@ -9384,7 +11286,7 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB194_1;
+; SM90-NEXT: @%p1 bra $L__BB259_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd26, 0;
; SM90-NEXT: {
@@ -9398,7 +11300,7 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
-; SM90-NEXT: $L__BB194_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB259_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.NaN.f32 %r13, %r21, %r5;
; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
@@ -9435,7 +11337,7 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
-; SM90-NEXT: @%p2 bra $L__BB194_3;
+; SM90-NEXT: @%p2 bra $L__BB259_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -9445,6 +11347,33 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x float> %retval
}
+define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v1f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB260_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f32 %r3, %r4, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM90-NEXT: mov.b32 %r4, %r2;
+; SM90-NEXT: @%p1 bra $L__BB260_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM90-LABEL: fmaximum_acq_rel_v2f32_global_cta(
; SM90: {
@@ -9459,7 +11388,7 @@ define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB195_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB261_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.NaN.f32 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -9475,7 +11404,7 @@ define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB195_1;
+; SM90-NEXT: @%p1 bra $L__BB261_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -9507,7 +11436,7 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB196_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB262_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.NaN.f32 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -9544,7 +11473,7 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB196_1;
+; SM90-NEXT: @%p1 bra $L__BB262_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -9577,7 +11506,7 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB197_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB263_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.NaN.f32 %r9, %r17, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
@@ -9614,7 +11543,7 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB197_1;
+; SM90-NEXT: @%p1 bra $L__BB263_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd26, 0;
; SM90-NEXT: {
@@ -9628,7 +11557,7 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
-; SM90-NEXT: $L__BB197_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB263_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.NaN.f32 %r13, %r21, %r5;
; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
@@ -9665,7 +11594,7 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
-; SM90-NEXT: @%p2 bra $L__BB197_3;
+; SM90-NEXT: @%p2 bra $L__BB263_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -9675,6 +11604,23 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
ret <8 x float> %retval
}
+define <1 x double> @fadd_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM90-LABEL: fadd_acq_rel_v1f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fadd_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM90-LABEL: fadd_acq_rel_v2f64_global_cta(
; SM90: {
@@ -9745,6 +11691,32 @@ define <8 x double> @fadd_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x double> %retval
}
+define <1 x double> @fsub_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM90-LABEL: fsub_acq_rel_v1f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [fsub_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM90-NEXT: $L__BB268_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f64 %rd4, %rd5, %rd1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM90-NEXT: mov.b64 %rd5, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB268_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM90-LABEL: fsub_acq_rel_v2f64_global_cta(
; SM90: {
@@ -9763,7 +11735,7 @@ define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
; SM90-NEXT: mov.b128 {%rd12, %rd13}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB201_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB269_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.f64 %rd7, %rd12, %rd4;
; SM90-NEXT: sub.rn.f64 %rd8, %rd13, %rd5;
@@ -9780,7 +11752,7 @@ define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: setp.ne.b64 %p1, %rd11, 0;
; SM90-NEXT: mov.b64 %rd12, %rd1;
; SM90-NEXT: mov.b64 %rd13, %rd2;
-; SM90-NEXT: @%p1 bra $L__BB201_1;
+; SM90-NEXT: @%p1 bra $L__BB269_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
@@ -9808,7 +11780,7 @@ define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
; SM90-NEXT: mov.b128 {%rd22, %rd23}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB202_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB270_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.f64 %rd11, %rd22, %rd1;
; SM90-NEXT: sub.rn.f64 %rd12, %rd23, %rd2;
@@ -9825,7 +11797,7 @@ define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.ne.b64 %p1, %rd15, 0;
; SM90-NEXT: mov.b64 %rd22, %rd5;
; SM90-NEXT: mov.b64 %rd23, %rd6;
-; SM90-NEXT: @%p1 bra $L__BB202_1;
+; SM90-NEXT: @%p1 bra $L__BB270_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd16, 0;
; SM90-NEXT: {
@@ -9835,7 +11807,7 @@ define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd24, %rd25}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB202_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB270_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.f64 %rd17, %rd24, %rd3;
; SM90-NEXT: sub.rn.f64 %rd18, %rd25, %rd4;
@@ -9852,7 +11824,7 @@ define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.ne.b64 %p2, %rd21, 0;
; SM90-NEXT: mov.b64 %rd24, %rd7;
; SM90-NEXT: mov.b64 %rd25, %rd8;
-; SM90-NEXT: @%p2 bra $L__BB202_3;
+; SM90-NEXT: @%p2 bra $L__BB270_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -9883,7 +11855,7 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB203_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB271_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd10, %rd43;
; SM90-NEXT: mov.b64 %rd9, %rd42;
@@ -9900,7 +11872,7 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: xor.b64 %rd22, %rd42, %rd9;
; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd23, 0;
-; SM90-NEXT: @%p1 bra $L__BB203_1;
+; SM90-NEXT: @%p1 bra $L__BB271_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd24, 0;
; SM90-NEXT: {
@@ -9910,7 +11882,7 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB203_3: // %atomicrmw.start6
+; SM90-NEXT: $L__BB271_3: // %atomicrmw.start6
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd12, %rd45;
; SM90-NEXT: mov.b64 %rd11, %rd44;
@@ -9927,7 +11899,7 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: xor.b64 %rd28, %rd44, %rd11;
; SM90-NEXT: or.b64 %rd29, %rd28, %rd27;
; SM90-NEXT: setp.ne.b64 %p2, %rd29, 0;
-; SM90-NEXT: @%p2 bra $L__BB203_3;
+; SM90-NEXT: @%p2 bra $L__BB271_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end5
; SM90-NEXT: mov.b64 %rd30, 0;
; SM90-NEXT: {
@@ -9937,7 +11909,7 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
; SM90-NEXT: mov.b128 {%rd46, %rd47}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB203_5: // %atomicrmw.start16
+; SM90-NEXT: $L__BB271_5: // %atomicrmw.start16
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.f64 %rd31, %rd46, %rd1;
; SM90-NEXT: sub.rn.f64 %rd32, %rd47, %rd2;
@@ -9954,7 +11926,7 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: setp.ne.b64 %p3, %rd35, 0;
; SM90-NEXT: mov.b64 %rd46, %rd13;
; SM90-NEXT: mov.b64 %rd47, %rd14;
-; SM90-NEXT: @%p3 bra $L__BB203_5;
+; SM90-NEXT: @%p3 bra $L__BB271_5;
; SM90-NEXT: // %bb.6: // %atomicrmw.end15
; SM90-NEXT: mov.b64 %rd36, 0;
; SM90-NEXT: {
@@ -9964,7 +11936,7 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
; SM90-NEXT: mov.b128 {%rd48, %rd49}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB203_7: // %atomicrmw.start22
+; SM90-NEXT: $L__BB271_7: // %atomicrmw.start22
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.f64 %rd37, %rd48, %rd3;
; SM90-NEXT: sub.rn.f64 %rd38, %rd49, %rd4;
@@ -9981,7 +11953,7 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: setp.ne.b64 %p4, %rd41, 0;
; SM90-NEXT: mov.b64 %rd48, %rd15;
; SM90-NEXT: mov.b64 %rd49, %rd16;
-; SM90-NEXT: @%p4 bra $L__BB203_7;
+; SM90-NEXT: @%p4 bra $L__BB271_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end21
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -9993,6 +11965,32 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x double> %retval
}
+define <1 x double> @fmin_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM90-LABEL: fmin_acq_rel_v1f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM90-NEXT: $L__BB272_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f64 %rd4, %rd5, %rd1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM90-NEXT: mov.b64 %rd5, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB272_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM90-LABEL: fmin_acq_rel_v2f64_global_cta(
; SM90: {
@@ -10011,7 +12009,7 @@ define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
; SM90-NEXT: mov.b128 {%rd12, %rd13}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB204_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB273_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.f64 %rd7, %rd12, %rd4;
; SM90-NEXT: min.f64 %rd8, %rd13, %rd5;
@@ -10028,7 +12026,7 @@ define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: setp.ne.b64 %p1, %rd11, 0;
; SM90-NEXT: mov.b64 %rd12, %rd1;
; SM90-NEXT: mov.b64 %rd13, %rd2;
-; SM90-NEXT: @%p1 bra $L__BB204_1;
+; SM90-NEXT: @%p1 bra $L__BB273_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
@@ -10056,7 +12054,7 @@ define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
; SM90-NEXT: mov.b128 {%rd22, %rd23}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB205_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB274_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.f64 %rd11, %rd22, %rd1;
; SM90-NEXT: min.f64 %rd12, %rd23, %rd2;
@@ -10073,7 +12071,7 @@ define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.ne.b64 %p1, %rd15, 0;
; SM90-NEXT: mov.b64 %rd22, %rd5;
; SM90-NEXT: mov.b64 %rd23, %rd6;
-; SM90-NEXT: @%p1 bra $L__BB205_1;
+; SM90-NEXT: @%p1 bra $L__BB274_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd16, 0;
; SM90-NEXT: {
@@ -10083,7 +12081,7 @@ define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd24, %rd25}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB205_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB274_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.f64 %rd17, %rd24, %rd3;
; SM90-NEXT: min.f64 %rd18, %rd25, %rd4;
@@ -10100,7 +12098,7 @@ define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.ne.b64 %p2, %rd21, 0;
; SM90-NEXT: mov.b64 %rd24, %rd7;
; SM90-NEXT: mov.b64 %rd25, %rd8;
-; SM90-NEXT: @%p2 bra $L__BB205_3;
+; SM90-NEXT: @%p2 bra $L__BB274_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -10131,7 +12129,7 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB206_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB275_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd10, %rd43;
; SM90-NEXT: mov.b64 %rd9, %rd42;
@@ -10148,7 +12146,7 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: xor.b64 %rd22, %rd42, %rd9;
; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd23, 0;
-; SM90-NEXT: @%p1 bra $L__BB206_1;
+; SM90-NEXT: @%p1 bra $L__BB275_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd24, 0;
; SM90-NEXT: {
@@ -10158,7 +12156,7 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB206_3: // %atomicrmw.start6
+; SM90-NEXT: $L__BB275_3: // %atomicrmw.start6
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd12, %rd45;
; SM90-NEXT: mov.b64 %rd11, %rd44;
@@ -10175,7 +12173,7 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: xor.b64 %rd28, %rd44, %rd11;
; SM90-NEXT: or.b64 %rd29, %rd28, %rd27;
; SM90-NEXT: setp.ne.b64 %p2, %rd29, 0;
-; SM90-NEXT: @%p2 bra $L__BB206_3;
+; SM90-NEXT: @%p2 bra $L__BB275_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end5
; SM90-NEXT: mov.b64 %rd30, 0;
; SM90-NEXT: {
@@ -10185,7 +12183,7 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
; SM90-NEXT: mov.b128 {%rd46, %rd47}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB206_5: // %atomicrmw.start15
+; SM90-NEXT: $L__BB275_5: // %atomicrmw.start15
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.f64 %rd31, %rd46, %rd1;
; SM90-NEXT: min.f64 %rd32, %rd47, %rd2;
@@ -10202,7 +12200,7 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: setp.ne.b64 %p3, %rd35, 0;
; SM90-NEXT: mov.b64 %rd46, %rd13;
; SM90-NEXT: mov.b64 %rd47, %rd14;
-; SM90-NEXT: @%p3 bra $L__BB206_5;
+; SM90-NEXT: @%p3 bra $L__BB275_5;
; SM90-NEXT: // %bb.6: // %atomicrmw.end14
; SM90-NEXT: mov.b64 %rd36, 0;
; SM90-NEXT: {
@@ -10212,7 +12210,7 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
; SM90-NEXT: mov.b128 {%rd48, %rd49}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB206_7: // %atomicrmw.start20
+; SM90-NEXT: $L__BB275_7: // %atomicrmw.start20
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.f64 %rd37, %rd48, %rd3;
; SM90-NEXT: min.f64 %rd38, %rd49, %rd4;
@@ -10229,7 +12227,7 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: setp.ne.b64 %p4, %rd41, 0;
; SM90-NEXT: mov.b64 %rd48, %rd15;
; SM90-NEXT: mov.b64 %rd49, %rd16;
-; SM90-NEXT: @%p4 bra $L__BB206_7;
+; SM90-NEXT: @%p4 bra $L__BB275_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end19
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -10241,6 +12239,32 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x double> %retval
}
+define <1 x double> @fmax_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM90-LABEL: fmax_acq_rel_v1f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM90-NEXT: $L__BB276_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f64 %rd4, %rd5, %rd1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM90-NEXT: mov.b64 %rd5, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB276_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM90-LABEL: fmax_acq_rel_v2f64_global_cta(
; SM90: {
@@ -10259,7 +12283,7 @@ define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
; SM90-NEXT: mov.b128 {%rd12, %rd13}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB207_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB277_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.f64 %rd7, %rd12, %rd4;
; SM90-NEXT: max.f64 %rd8, %rd13, %rd5;
@@ -10276,7 +12300,7 @@ define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: setp.ne.b64 %p1, %rd11, 0;
; SM90-NEXT: mov.b64 %rd12, %rd1;
; SM90-NEXT: mov.b64 %rd13, %rd2;
-; SM90-NEXT: @%p1 bra $L__BB207_1;
+; SM90-NEXT: @%p1 bra $L__BB277_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
@@ -10304,7 +12328,7 @@ define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
; SM90-NEXT: mov.b128 {%rd22, %rd23}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB208_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB278_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.f64 %rd11, %rd22, %rd1;
; SM90-NEXT: max.f64 %rd12, %rd23, %rd2;
@@ -10321,7 +12345,7 @@ define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.ne.b64 %p1, %rd15, 0;
; SM90-NEXT: mov.b64 %rd22, %rd5;
; SM90-NEXT: mov.b64 %rd23, %rd6;
-; SM90-NEXT: @%p1 bra $L__BB208_1;
+; SM90-NEXT: @%p1 bra $L__BB278_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd16, 0;
; SM90-NEXT: {
@@ -10331,7 +12355,7 @@ define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd24, %rd25}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB208_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB278_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.f64 %rd17, %rd24, %rd3;
; SM90-NEXT: max.f64 %rd18, %rd25, %rd4;
@@ -10348,7 +12372,7 @@ define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.ne.b64 %p2, %rd21, 0;
; SM90-NEXT: mov.b64 %rd24, %rd7;
; SM90-NEXT: mov.b64 %rd25, %rd8;
-; SM90-NEXT: @%p2 bra $L__BB208_3;
+; SM90-NEXT: @%p2 bra $L__BB278_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -10379,7 +12403,7 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB209_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB279_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd10, %rd43;
; SM90-NEXT: mov.b64 %rd9, %rd42;
@@ -10396,7 +12420,7 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: xor.b64 %rd22, %rd42, %rd9;
; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd23, 0;
-; SM90-NEXT: @%p1 bra $L__BB209_1;
+; SM90-NEXT: @%p1 bra $L__BB279_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd24, 0;
; SM90-NEXT: {
@@ -10406,7 +12430,7 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB209_3: // %atomicrmw.start6
+; SM90-NEXT: $L__BB279_3: // %atomicrmw.start6
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd12, %rd45;
; SM90-NEXT: mov.b64 %rd11, %rd44;
@@ -10423,7 +12447,7 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: xor.b64 %rd28, %rd44, %rd11;
; SM90-NEXT: or.b64 %rd29, %rd28, %rd27;
; SM90-NEXT: setp.ne.b64 %p2, %rd29, 0;
-; SM90-NEXT: @%p2 bra $L__BB209_3;
+; SM90-NEXT: @%p2 bra $L__BB279_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end5
; SM90-NEXT: mov.b64 %rd30, 0;
; SM90-NEXT: {
@@ -10433,7 +12457,7 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
; SM90-NEXT: mov.b128 {%rd46, %rd47}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB209_5: // %atomicrmw.start15
+; SM90-NEXT: $L__BB279_5: // %atomicrmw.start15
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.f64 %rd31, %rd46, %rd1;
; SM90-NEXT: max.f64 %rd32, %rd47, %rd2;
@@ -10450,7 +12474,7 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: setp.ne.b64 %p3, %rd35, 0;
; SM90-NEXT: mov.b64 %rd46, %rd13;
; SM90-NEXT: mov.b64 %rd47, %rd14;
-; SM90-NEXT: @%p3 bra $L__BB209_5;
+; SM90-NEXT: @%p3 bra $L__BB279_5;
; SM90-NEXT: // %bb.6: // %atomicrmw.end14
; SM90-NEXT: mov.b64 %rd36, 0;
; SM90-NEXT: {
@@ -10460,7 +12484,7 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
; SM90-NEXT: mov.b128 {%rd48, %rd49}, dst;
; SM90-NEXT: }
-; SM90-NEXT: $L__BB209_7: // %atomicrmw.start20
+; SM90-NEXT: $L__BB279_7: // %atomicrmw.start20
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.f64 %rd37, %rd48, %rd3;
; SM90-NEXT: max.f64 %rd38, %rd49, %rd4;
@@ -10477,7 +12501,7 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: setp.ne.b64 %p4, %rd41, 0;
; SM90-NEXT: mov.b64 %rd48, %rd15;
; SM90-NEXT: mov.b64 %rd49, %rd16;
-; SM90-NEXT: @%p4 bra $L__BB209_7;
+; SM90-NEXT: @%p4 bra $L__BB279_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end19
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -10489,6 +12513,40 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x double> %retval
}
+define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM90-LABEL: fminimum_acq_rel_v1f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd3];
+; SM90-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM90-NEXT: $L__BB280_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
+; SM90-NEXT: min.f64 %rd4, %rd9, %rd1;
+; SM90-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd9, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
+; SM90-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
+; SM90-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
+; SM90-NEXT: mov.b64 %rd9, %rd2;
+; SM90-NEXT: @%p5 bra $L__BB280_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM90-LABEL: fminimum_acq_rel_v2f64_global_cta(
; SM90: {
@@ -10509,7 +12567,7 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p3, %rd4, -9223372036854775808;
; SM90-NEXT: setp.eq.b64 %p7, %rd5, -9223372036854775808;
-; SM90-NEXT: $L__BB210_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB281_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.nan.f64 %p1, %rd20, %rd4;
; SM90-NEXT: min.f64 %rd7, %rd20, %rd4;
@@ -10540,7 +12598,7 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: setp.ne.b64 %p9, %rd19, 0;
; SM90-NEXT: mov.b64 %rd20, %rd1;
; SM90-NEXT: mov.b64 %rd21, %rd2;
-; SM90-NEXT: @%p9 bra $L__BB210_1;
+; SM90-NEXT: @%p9 bra $L__BB281_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
@@ -10570,7 +12628,7 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
; SM90-NEXT: setp.eq.b64 %p7, %rd2, -9223372036854775808;
-; SM90-NEXT: $L__BB211_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB282_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.nan.f64 %p1, %rd38, %rd1;
; SM90-NEXT: min.f64 %rd11, %rd38, %rd1;
@@ -10601,7 +12659,7 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: setp.ne.b64 %p9, %rd23, 0;
; SM90-NEXT: mov.b64 %rd38, %rd5;
; SM90-NEXT: mov.b64 %rd39, %rd6;
-; SM90-NEXT: @%p9 bra $L__BB211_1;
+; SM90-NEXT: @%p9 bra $L__BB282_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd24, 0;
; SM90-NEXT: {
@@ -10613,7 +12671,7 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p12, %rd3, -9223372036854775808;
; SM90-NEXT: setp.eq.b64 %p16, %rd4, -9223372036854775808;
-; SM90-NEXT: $L__BB211_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB282_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.nan.f64 %p10, %rd40, %rd3;
; SM90-NEXT: min.f64 %rd25, %rd40, %rd3;
@@ -10644,7 +12702,7 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: setp.ne.b64 %p18, %rd37, 0;
; SM90-NEXT: mov.b64 %rd40, %rd7;
; SM90-NEXT: mov.b64 %rd41, %rd8;
-; SM90-NEXT: @%p18 bra $L__BB211_3;
+; SM90-NEXT: @%p18 bra $L__BB282_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -10677,7 +12735,7 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p3, %rd5, -9223372036854775808;
; SM90-NEXT: setp.eq.b64 %p7, %rd6, -9223372036854775808;
-; SM90-NEXT: $L__BB212_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB283_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd10, %rd75;
; SM90-NEXT: mov.b64 %rd9, %rd74;
@@ -10708,7 +12766,7 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: xor.b64 %rd30, %rd74, %rd9;
; SM90-NEXT: or.b64 %rd31, %rd30, %rd29;
; SM90-NEXT: setp.ne.b64 %p9, %rd31, 0;
-; SM90-NEXT: @%p9 bra $L__BB212_1;
+; SM90-NEXT: @%p9 bra $L__BB283_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd32, 0;
; SM90-NEXT: {
@@ -10720,7 +12778,7 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p12, %rd7, -9223372036854775808;
; SM90-NEXT: setp.eq.b64 %p16, %rd8, -9223372036854775808;
-; SM90-NEXT: $L__BB212_3: // %atomicrmw.start6
+; SM90-NEXT: $L__BB283_3: // %atomicrmw.start6
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd12, %rd77;
; SM90-NEXT: mov.b64 %rd11, %rd76;
@@ -10751,7 +12809,7 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: xor.b64 %rd44, %rd76, %rd11;
; SM90-NEXT: or.b64 %rd45, %rd44, %rd43;
; SM90-NEXT: setp.ne.b64 %p18, %rd45, 0;
-; SM90-NEXT: @%p18 bra $L__BB212_3;
+; SM90-NEXT: @%p18 bra $L__BB283_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end5
; SM90-NEXT: mov.b64 %rd46, 0;
; SM90-NEXT: {
@@ -10763,7 +12821,7 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p21, %rd1, -9223372036854775808;
; SM90-NEXT: setp.eq.b64 %p25, %rd2, -9223372036854775808;
-; SM90-NEXT: $L__BB212_5: // %atomicrmw.start15
+; SM90-NEXT: $L__BB283_5: // %atomicrmw.start15
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.nan.f64 %p19, %rd78, %rd1;
; SM90-NEXT: min.f64 %rd47, %rd78, %rd1;
@@ -10794,7 +12852,7 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: setp.ne.b64 %p27, %rd59, 0;
; SM90-NEXT: mov.b64 %rd78, %rd13;
; SM90-NEXT: mov.b64 %rd79, %rd14;
-; SM90-NEXT: @%p27 bra $L__BB212_5;
+; SM90-NEXT: @%p27 bra $L__BB283_5;
; SM90-NEXT: // %bb.6: // %atomicrmw.end14
; SM90-NEXT: mov.b64 %rd60, 0;
; SM90-NEXT: {
@@ -10806,7 +12864,7 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p30, %rd3, -9223372036854775808;
; SM90-NEXT: setp.eq.b64 %p34, %rd4, -9223372036854775808;
-; SM90-NEXT: $L__BB212_7: // %atomicrmw.start20
+; SM90-NEXT: $L__BB283_7: // %atomicrmw.start20
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.nan.f64 %p28, %rd80, %rd3;
; SM90-NEXT: min.f64 %rd61, %rd80, %rd3;
@@ -10837,7 +12895,7 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: setp.ne.b64 %p36, %rd73, 0;
; SM90-NEXT: mov.b64 %rd80, %rd15;
; SM90-NEXT: mov.b64 %rd81, %rd16;
-; SM90-NEXT: @%p36 bra $L__BB212_7;
+; SM90-NEXT: @%p36 bra $L__BB283_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end19
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -10849,6 +12907,40 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
ret <8 x double> %retval
}
+define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v1f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd3];
+; SM90-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM90-NEXT: $L__BB284_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
+; SM90-NEXT: max.f64 %rd4, %rd9, %rd1;
+; SM90-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd9, 0;
+; SM90-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
+; SM90-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
+; SM90-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
+; SM90-NEXT: mov.b64 %rd9, %rd2;
+; SM90-NEXT: @%p5 bra $L__BB284_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM90-LABEL: fmaximum_acq_rel_v2f64_global_cta(
; SM90: {
@@ -10869,7 +12961,7 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p3, %rd4, 0;
; SM90-NEXT: setp.eq.b64 %p7, %rd5, 0;
-; SM90-NEXT: $L__BB213_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB285_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.nan.f64 %p1, %rd20, %rd4;
; SM90-NEXT: max.f64 %rd7, %rd20, %rd4;
@@ -10900,7 +12992,7 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: setp.ne.b64 %p9, %rd19, 0;
; SM90-NEXT: mov.b64 %rd20, %rd1;
; SM90-NEXT: mov.b64 %rd21, %rd2;
-; SM90-NEXT: @%p9 bra $L__BB213_1;
+; SM90-NEXT: @%p9 bra $L__BB285_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
@@ -10930,7 +13022,7 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p3, %rd1, 0;
; SM90-NEXT: setp.eq.b64 %p7, %rd2, 0;
-; SM90-NEXT: $L__BB214_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB286_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.nan.f64 %p1, %rd38, %rd1;
; SM90-NEXT: max.f64 %rd11, %rd38, %rd1;
@@ -10961,7 +13053,7 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: setp.ne.b64 %p9, %rd23, 0;
; SM90-NEXT: mov.b64 %rd38, %rd5;
; SM90-NEXT: mov.b64 %rd39, %rd6;
-; SM90-NEXT: @%p9 bra $L__BB214_1;
+; SM90-NEXT: @%p9 bra $L__BB286_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd24, 0;
; SM90-NEXT: {
@@ -10973,7 +13065,7 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p12, %rd3, 0;
; SM90-NEXT: setp.eq.b64 %p16, %rd4, 0;
-; SM90-NEXT: $L__BB214_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB286_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.nan.f64 %p10, %rd40, %rd3;
; SM90-NEXT: max.f64 %rd25, %rd40, %rd3;
@@ -11004,7 +13096,7 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: setp.ne.b64 %p18, %rd37, 0;
; SM90-NEXT: mov.b64 %rd40, %rd7;
; SM90-NEXT: mov.b64 %rd41, %rd8;
-; SM90-NEXT: @%p18 bra $L__BB214_3;
+; SM90-NEXT: @%p18 bra $L__BB286_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -11037,7 +13129,7 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p3, %rd5, 0;
; SM90-NEXT: setp.eq.b64 %p7, %rd6, 0;
-; SM90-NEXT: $L__BB215_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB287_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd10, %rd75;
; SM90-NEXT: mov.b64 %rd9, %rd74;
@@ -11068,7 +13160,7 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: xor.b64 %rd30, %rd74, %rd9;
; SM90-NEXT: or.b64 %rd31, %rd30, %rd29;
; SM90-NEXT: setp.ne.b64 %p9, %rd31, 0;
-; SM90-NEXT: @%p9 bra $L__BB215_1;
+; SM90-NEXT: @%p9 bra $L__BB287_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd32, 0;
; SM90-NEXT: {
@@ -11080,7 +13172,7 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p12, %rd7, 0;
; SM90-NEXT: setp.eq.b64 %p16, %rd8, 0;
-; SM90-NEXT: $L__BB215_3: // %atomicrmw.start6
+; SM90-NEXT: $L__BB287_3: // %atomicrmw.start6
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: mov.b64 %rd12, %rd77;
; SM90-NEXT: mov.b64 %rd11, %rd76;
@@ -11111,7 +13203,7 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: xor.b64 %rd44, %rd76, %rd11;
; SM90-NEXT: or.b64 %rd45, %rd44, %rd43;
; SM90-NEXT: setp.ne.b64 %p18, %rd45, 0;
-; SM90-NEXT: @%p18 bra $L__BB215_3;
+; SM90-NEXT: @%p18 bra $L__BB287_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end5
; SM90-NEXT: mov.b64 %rd46, 0;
; SM90-NEXT: {
@@ -11123,7 +13215,7 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p21, %rd1, 0;
; SM90-NEXT: setp.eq.b64 %p25, %rd2, 0;
-; SM90-NEXT: $L__BB215_5: // %atomicrmw.start15
+; SM90-NEXT: $L__BB287_5: // %atomicrmw.start15
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.nan.f64 %p19, %rd78, %rd1;
; SM90-NEXT: max.f64 %rd47, %rd78, %rd1;
@@ -11154,7 +13246,7 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: setp.ne.b64 %p27, %rd59, 0;
; SM90-NEXT: mov.b64 %rd78, %rd13;
; SM90-NEXT: mov.b64 %rd79, %rd14;
-; SM90-NEXT: @%p27 bra $L__BB215_5;
+; SM90-NEXT: @%p27 bra $L__BB287_5;
; SM90-NEXT: // %bb.6: // %atomicrmw.end14
; SM90-NEXT: mov.b64 %rd60, 0;
; SM90-NEXT: {
@@ -11166,7 +13258,7 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: }
; SM90-NEXT: setp.eq.b64 %p30, %rd3, 0;
; SM90-NEXT: setp.eq.b64 %p34, %rd4, 0;
-; SM90-NEXT: $L__BB215_7: // %atomicrmw.start20
+; SM90-NEXT: $L__BB287_7: // %atomicrmw.start20
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: setp.nan.f64 %p28, %rd80, %rd3;
; SM90-NEXT: max.f64 %rd61, %rd80, %rd3;
@@ -11197,7 +13289,7 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: setp.ne.b64 %p36, %rd73, 0;
; SM90-NEXT: mov.b64 %rd80, %rd15;
; SM90-NEXT: mov.b64 %rd81, %rd16;
-; SM90-NEXT: @%p36 bra $L__BB215_7;
+; SM90-NEXT: @%p36 bra $L__BB287_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end19
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
@@ -11209,6 +13301,24 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
ret <8 x double> %retval
}
+define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-LABEL: fadd_acq_rel_v1f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM90-LABEL: fadd_acq_rel_v2f16_global_cta(
; SM90: {
@@ -11283,6 +13393,48 @@ define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
ret <8 x half> %retval
}
+define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-LABEL: fsub_acq_rel_v1f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB292_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB292_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM90-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM90: {
@@ -11295,13 +13447,13 @@ define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB219_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB293_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.f16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB219_1;
+; SM90-NEXT: @%p1 bra $L__BB293_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -11324,7 +13476,7 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB220_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB294_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.f16x2 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -11340,7 +13492,7 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB220_1;
+; SM90-NEXT: @%p1 bra $L__BB294_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -11372,7 +13524,7 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB221_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB295_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.f16x2 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -11409,7 +13561,7 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB221_1;
+; SM90-NEXT: @%p1 bra $L__BB295_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -11418,6 +13570,48 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
ret <8 x half> %retval
}
+define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-LABEL: fmin_acq_rel_v1f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB296_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: min.f16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB296_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM90-LABEL: fmin_acq_rel_v2f16_global_cta(
; SM90: {
@@ -11430,13 +13624,13 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB222_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB297_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.f16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB222_1;
+; SM90-NEXT: @%p1 bra $L__BB297_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -11459,7 +13653,7 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB223_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB298_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.f16x2 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -11475,7 +13669,7 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB223_1;
+; SM90-NEXT: @%p1 bra $L__BB298_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -11507,7 +13701,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB224_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB299_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.f16x2 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -11544,7 +13738,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB224_1;
+; SM90-NEXT: @%p1 bra $L__BB299_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -11553,6 +13747,48 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
ret <8 x half> %retval
}
+define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v1f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB300_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.f16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB300_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM90-LABEL: fmax_acq_rel_v2f16_global_cta(
; SM90: {
@@ -11565,13 +13801,13 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB225_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB301_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.f16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB225_1;
+; SM90-NEXT: @%p1 bra $L__BB301_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -11594,7 +13830,7 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB226_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB302_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.f16x2 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -11610,7 +13846,7 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB226_1;
+; SM90-NEXT: @%p1 bra $L__BB302_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -11642,7 +13878,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB227_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB303_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.f16x2 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -11679,7 +13915,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB227_1;
+; SM90-NEXT: @%p1 bra $L__BB303_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -11688,6 +13924,48 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
ret <8 x half> %retval
}
+define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-LABEL: fminimum_acq_rel_v1f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB304_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: min.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB304_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM90-LABEL: fminimum_acq_rel_v2f16_global_cta(
; SM90: {
@@ -11700,13 +13978,13 @@ define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB228_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB305_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.NaN.f16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB228_1;
+; SM90-NEXT: @%p1 bra $L__BB305_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -11729,7 +14007,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB229_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB306_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.NaN.f16x2 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -11745,7 +14023,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB229_1;
+; SM90-NEXT: @%p1 bra $L__BB306_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -11777,7 +14055,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB230_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB307_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.NaN.f16x2 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -11814,7 +14092,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB230_1;
+; SM90-NEXT: @%p1 bra $L__BB307_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -11823,6 +14101,48 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x half> %retval
}
+define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v1f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB308_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB308_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM90-LABEL: fmaximum_acq_rel_v2f16_global_cta(
; SM90: {
@@ -11835,13 +14155,13 @@ define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB231_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB309_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.NaN.f16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB231_1;
+; SM90-NEXT: @%p1 bra $L__BB309_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -11864,7 +14184,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB232_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB310_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.NaN.f16x2 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -11880,7 +14200,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB232_1;
+; SM90-NEXT: @%p1 bra $L__BB310_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -11912,7 +14232,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB233_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB311_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.NaN.f16x2 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -11949,7 +14269,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB233_1;
+; SM90-NEXT: @%p1 bra $L__BB311_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -11958,6 +14278,24 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
ret <8 x half> %retval
}
+define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM90-LABEL: fadd_acq_rel_v1bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fadd_acq_rel_v2bf16_global_cta(
; SM90: {
@@ -12032,6 +14370,48 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x bfloat> %retval
}
+define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM90-LABEL: fsub_acq_rel_v1bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB316_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: sub.rn.bf16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB316_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fsub_acq_rel_v2bf16_global_cta(
; SM90: {
@@ -12044,13 +14424,13 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB237_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB317_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.bf16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB237_1;
+; SM90-NEXT: @%p1 bra $L__BB317_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -12073,7 +14453,7 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB238_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB318_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.bf16x2 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -12089,7 +14469,7 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB238_1;
+; SM90-NEXT: @%p1 bra $L__BB318_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -12121,7 +14501,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB239_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB319_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: sub.rn.bf16x2 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -12158,7 +14538,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB239_1;
+; SM90-NEXT: @%p1 bra $L__BB319_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -12167,6 +14547,48 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x bfloat> %retval
}
+define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM90-LABEL: fmin_acq_rel_v1bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB320_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: min.bf16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB320_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fmin_acq_rel_v2bf16_global_cta(
; SM90: {
@@ -12179,13 +14601,13 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB240_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB321_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.bf16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB240_1;
+; SM90-NEXT: @%p1 bra $L__BB321_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -12208,7 +14630,7 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB241_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB322_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.bf16x2 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -12224,7 +14646,7 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB241_1;
+; SM90-NEXT: @%p1 bra $L__BB322_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -12256,7 +14678,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB242_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB323_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.bf16x2 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -12293,7 +14715,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB242_1;
+; SM90-NEXT: @%p1 bra $L__BB323_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -12302,6 +14724,48 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x bfloat> %retval
}
+define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM90-LABEL: fmax_acq_rel_v1bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB324_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.bf16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB324_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fmax_acq_rel_v2bf16_global_cta(
; SM90: {
@@ -12314,13 +14778,13 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB243_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB325_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.bf16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB243_1;
+; SM90-NEXT: @%p1 bra $L__BB325_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -12343,7 +14807,7 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB244_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB326_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.bf16x2 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -12359,7 +14823,7 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB244_1;
+; SM90-NEXT: @%p1 bra $L__BB326_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -12391,7 +14855,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB245_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB327_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.bf16x2 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -12428,7 +14892,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB245_1;
+; SM90-NEXT: @%p1 bra $L__BB327_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -12437,6 +14901,48 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x bfloat> %retval
}
+define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM90-LABEL: fminimum_acq_rel_v1bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB328_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: min.NaN.bf16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB328_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fminimum_acq_rel_v2bf16_global_cta(
; SM90: {
@@ -12449,13 +14955,13 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB246_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB329_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.NaN.bf16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB246_1;
+; SM90-NEXT: @%p1 bra $L__BB329_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -12478,7 +14984,7 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB247_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB330_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.NaN.bf16x2 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -12494,7 +15000,7 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB247_1;
+; SM90-NEXT: @%p1 bra $L__BB330_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -12526,7 +15032,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB248_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB331_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: min.NaN.bf16x2 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -12563,7 +15069,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB248_1;
+; SM90-NEXT: @%p1 bra $L__BB331_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -12572,6 +15078,48 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
ret <8 x bfloat> %retval
}
+define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v1bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB332_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.NaN.bf16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB332_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM90-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
; SM90: {
@@ -12584,13 +15132,13 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB249_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB333_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.NaN.bf16x2 %r3, %r4, %r2;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB249_1;
+; SM90-NEXT: @%p1 bra $L__BB333_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -12613,7 +15161,7 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB250_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB334_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.NaN.bf16x2 %r3, %r5, %r1;
; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
@@ -12629,7 +15177,7 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB250_1;
+; SM90-NEXT: @%p1 bra $L__BB334_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -12661,7 +15209,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB251_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB335_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: max.NaN.bf16x2 %r5, %r9, %r1;
; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
@@ -12698,7 +15246,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
-; SM90-NEXT: @%p1 bra $L__BB251_1;
+; SM90-NEXT: @%p1 bra $L__BB335_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -12707,6 +15255,43 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
ret <8 x bfloat> %retval
}
+define <1 x i8> @add_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: add_monotonic_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [add_monotonic_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b8 %r6, [add_monotonic_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB336_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB336_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: add_monotonic_v2i8_global_cta(
; SM90: {
@@ -12727,7 +15312,7 @@ define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM90-NEXT: shl.b32 %r8, %r7, %r1;
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM90-NEXT: $L__BB252_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB337_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r17, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -12745,7 +15330,7 @@ define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
; SM90-NEXT: mov.b32 %r17, %r3;
-; SM90-NEXT: @%p1 bra $L__BB252_1;
+; SM90-NEXT: @%p1 bra $L__BB337_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r16, %r3, %r1;
; SM90-NEXT: st.param.b16 [func_retval0], %r16;
@@ -12754,104 +15339,188 @@ define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
ret <2 x i8> %retval
}
-define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM90-LABEL: add_acquire_v2i8_global_cta(
+define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_monotonic_v4i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<10>;
-; SM90-NEXT: .reg .b32 %r<18>;
-; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
-; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
-; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: and.b32 %r6, %r5, 3;
-; SM90-NEXT: shl.b32 %r1, %r6, 3;
-; SM90-NEXT: mov.b32 %r7, 65535;
-; SM90-NEXT: shl.b32 %r8, %r7, %r1;
-; SM90-NEXT: not.b32 %r2, %r8;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM90-NEXT: $L__BB253_1: // %atomicrmw.start
+; SM90-NEXT: ld.param.b32 %r2, [add_monotonic_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: $L__BB338_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r9, %r17, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
-; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM90-NEXT: and.b16 %rs8, %rs5, 255;
-; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM90-NEXT: shl.b32 %r13, %r12, %r1;
-; SM90-NEXT: and.b32 %r14, %r17, %r2;
-; SM90-NEXT: or.b32 %r15, %r14, %r13;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM90-NEXT: mov.b32 %r17, %r3;
-; SM90-NEXT: @%p1 bra $L__BB253_1;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB338_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r16, %r3, %r1;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
- ret <2 x i8> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
}
-define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM90-LABEL: add_release_v2i8_global_cta(
+define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: add_monotonic_v8i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<10>;
-; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB339_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB339_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @add_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: add_acquire_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
-; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [add_acquire_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b8 %r6, [add_acquire_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: and.b32 %r6, %r5, 3;
-; SM90-NEXT: shl.b32 %r1, %r6, 3;
-; SM90-NEXT: mov.b32 %r7, 65535;
-; SM90-NEXT: shl.b32 %r8, %r7, %r1;
-; SM90-NEXT: not.b32 %r2, %r8;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM90-NEXT: $L__BB254_1: // %atomicrmw.start
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB340_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r9, %r17, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
-; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM90-NEXT: and.b16 %rs8, %rs5, 255;
-; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
-; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM90-NEXT: shl.b32 %r13, %r12, %r1;
-; SM90-NEXT: and.b32 %r14, %r17, %r2;
-; SM90-NEXT: or.b32 %r15, %r14, %r13;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM90-NEXT: mov.b32 %r17, %r3;
-; SM90-NEXT: @%p1 bra $L__BB254_1;
+; SM90-NEXT: add.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB340_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r16, %r3, %r1;
-; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
- ret <2 x i8> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
+ ret <1 x i8> %retval
}
-define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM90-LABEL: add_seq_cst_v2i8_global_cta(
+define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: add_acquire_v2i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
; SM90-NEXT: .reg .b16 %rs<10>;
@@ -12859,10 +15528,9 @@ define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
; SM90-NEXT: and.b32 %r6, %r5, 3;
@@ -12871,7 +15539,7 @@ define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: shl.b32 %r8, %r7, %r1;
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM90-NEXT: $L__BB255_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB341_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r17, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -12889,18 +15557,18 @@ define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
; SM90-NEXT: mov.b32 %r17, %r3;
-; SM90-NEXT: @%p1 bra $L__BB255_1;
+; SM90-NEXT: @%p1 bra $L__BB341_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r16, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b16 [func_retval0], %r16;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
ret <2 x i8> %retval
}
-define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM90-LABEL: add_monotonic_v4i8_global_cta(
+define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_acquire_v4i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
; SM90-NEXT: .reg .b16 %rs<13>;
@@ -12908,8 +15576,8 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [add_monotonic_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
@@ -12919,7 +15587,7 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: $L__BB256_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB342_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -12943,122 +15611,194 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM90-NEXT: mov.b32 %r18, %r1;
-; SM90-NEXT: @%p1 bra $L__BB256_1;
+; SM90-NEXT: @%p1 bra $L__BB342_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
ret <4 x i8> %retval
}
-define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM90-LABEL: add_acquire_v4i8_global_cta(
+define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: add_acquire_v8i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<13>;
-; SM90-NEXT: .reg .b32 %r<19>;
-; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: $L__BB257_1: // %atomicrmw.start
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB343_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
-; SM90-NEXT: mov.b32 %r18, %r1;
-; SM90-NEXT: @%p1 bra $L__BB257_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB343_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
- ret <4 x i8> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
}
-define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM90-LABEL: add_release_v4i8_global_cta(
+define <1 x i8> @add_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: add_release_v1i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<13>;
-; SM90-NEXT: .reg .b32 %r<19>;
-; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b64 %rd2, [add_release_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: $L__BB258_1: // %atomicrmw.start
+; SM90-NEXT: ld.param.b8 %r6, [add_release_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB344_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
-; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
-; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
-; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
-; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
-; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
-; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: add.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB344_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: add_release_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<10>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM90-NEXT: $L__BB345_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: and.b16 %rs8, %rs5, 255;
+; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
-; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
-; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
-; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
-; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
-; SM90-NEXT: mov.b32 %r18, %r1;
-; SM90-NEXT: @%p1 bra $L__BB258_1;
+; SM90-NEXT: shl.b32 %r13, %r12, %r1;
+; SM90-NEXT: and.b32 %r14, %r17, %r2;
+; SM90-NEXT: or.b32 %r15, %r14, %r13;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM90-NEXT: mov.b32 %r17, %r3;
+; SM90-NEXT: @%p1 bra $L__BB345_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: shr.u32 %r16, %r3, %r1;
+; SM90-NEXT: st.param.b16 [func_retval0], %r16;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
- ret <4 x i8> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
}
-define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM90-LABEL: add_seq_cst_v4i8_global_cta(
+define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_release_v4i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
; SM90-NEXT: .reg .b16 %rs<13>;
@@ -13066,9 +15806,9 @@ define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
@@ -13078,7 +15818,7 @@ define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: $L__BB259_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB346_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -13102,17 +15842,16 @@ define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
; SM90-NEXT: mov.b32 %r18, %r1;
-; SM90-NEXT: @%p1 bra $L__BB259_1;
+; SM90-NEXT: @%p1 bra $L__BB346_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
ret <4 x i8> %retval
}
-define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM90-LABEL: add_monotonic_v8i8_global_cta(
+define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: add_release_v8i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
; SM90-NEXT: .reg .b16 %rs<25>;
@@ -13120,8 +15859,9 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
@@ -13135,7 +15875,7 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM90-NEXT: $L__BB260_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB347_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -13193,194 +15933,154 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB260_1;
+; SM90-NEXT: @%p1 bra $L__BB347_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
ret <8 x i8> %retval
}
-define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM90-LABEL: add_acquire_v8i8_global_cta(
+define <1 x i8> @add_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: add_seq_cst_v1i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<25>;
-; SM90-NEXT: .reg .b32 %r<35>;
-; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM90-NEXT: $L__BB261_1: // %atomicrmw.start
+; SM90-NEXT: ld.param.b64 %rd2, [add_seq_cst_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.b8 %r6, [add_seq_cst_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB348_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
-; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
-; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
-; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
-; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
-; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
-; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
-; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
-; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
-; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
-; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
-; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
-; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
-; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
-; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
-; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
-; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
-; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
-; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
-; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
-; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
-; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
-; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
-; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
-; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
-; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
-; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
-; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
-; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
-; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
-; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
-; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB261_1;
+; SM90-NEXT: add.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB348_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
- ret <8 x i8> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
+ ret <1 x i8> %retval
}
-define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM90-LABEL: add_release_v8i8_global_cta(
+define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: add_seq_cst_v2i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<25>;
-; SM90-NEXT: .reg .b32 %r<35>;
-; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-NEXT: .reg .b16 %rs<10>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_release_v8i8_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM90-NEXT: $L__BB349_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: and.b16 %rs8, %rs5, 255;
+; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: shl.b32 %r13, %r12, %r1;
+; SM90-NEXT: and.b32 %r14, %r17, %r2;
+; SM90-NEXT: or.b32 %r15, %r14, %r13;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM90-NEXT: mov.b32 %r17, %r3;
+; SM90-NEXT: @%p1 bra $L__BB349_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r16, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_seq_cst_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM90-NEXT: $L__BB262_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB350_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
-; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
-; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
-; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
-; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
-; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
-; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
-; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
-; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
-; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
-; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
-; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
-; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
-; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
-; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
-; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
-; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
-; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
-; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
-; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
-; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB262_1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB350_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
- ret <8 x i8> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
}
define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
@@ -13408,7 +16108,7 @@ define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM90-NEXT: $L__BB263_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB351_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
@@ -13466,7 +16166,7 @@ define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB263_1;
+; SM90-NEXT: @%p1 bra $L__BB351_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
@@ -13475,6 +16175,22 @@ define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
+define <1 x i32> @add_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: add_monotonic_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r1, [add_monotonic_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
+ ret <1 x i32> %retval
+}
+
define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM90-LABEL: add_monotonic_v2i32_global_cta(
; SM90: {
@@ -13492,78 +16208,83 @@ define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i3
ret <2 x i32> %retval
}
-define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM90-LABEL: add_acquire_v2i32_global_cta(
+define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: add_monotonic_v4i32_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b32 %r<9>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
- ret <2 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
}
-define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM90-LABEL: add_release_v2i32_global_cta(
+define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: add_monotonic_v8i32_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b32 %r<17>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
- ret <2 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
}
-define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM90-LABEL: add_seq_cst_v2i32_global_cta(
+define <1 x i32> @add_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: add_acquire_v1i32_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b32 %r<3>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r1, [add_acquire_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
- ret <2 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
+ ret <1 x i32> %retval
}
-define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM90-LABEL: add_monotonic_v4i32_global_cta(
+define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: add_acquire_v2i32_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b32 %r<5>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
- ret <4 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
}
define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
@@ -13586,57 +16307,16 @@ define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
ret <4 x i32> %retval
}
-define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM90-LABEL: add_release_v4i32_global_cta(
+define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: add_acquire_v8i32_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b32 %r<17>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
- ret <4 x i32> %retval
-}
-
-define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM90-LABEL: add_seq_cst_v4i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .b32 %r<9>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
- ret <4 x i32> %retval
-}
-
-define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM90-LABEL: add_monotonic_v8i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .b32 %r<17>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
@@ -13645,37 +16325,67 @@ define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i3
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
ret <8 x i32> %retval
}
-define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM90-LABEL: add_acquire_v8i32_global_cta(
+define <1 x i32> @add_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: add_release_v1i32_global_cta(
; SM90: {
-; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b32 %r<3>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [add_release_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
- ret <8 x i32> %retval
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: add_release_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: add_release_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
}
define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
@@ -13704,6 +16414,64 @@ define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i32> @add_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: add_seq_cst_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.b32 %r1, [add_seq_cst_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: add_seq_cst_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: add_seq_cst_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM90-LABEL: add_seq_cst_v8i32_global_cta(
; SM90: {
@@ -13731,6 +16499,44 @@ define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
+define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: nand_monotonic_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [nand_monotonic_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b8 %r6, [nand_monotonic_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM90-NEXT: $L__BB368_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
+; SM90-NEXT: @%p1 bra $L__BB368_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
+; SM90-NEXT: st.param.b8 [func_retval0], %r15;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: nand_monotonic_v2i8_global_cta(
; SM90: {
@@ -13751,7 +16557,7 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM90-NEXT: shl.b32 %r8, %r7, %r1;
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: $L__BB276_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB369_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r18, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -13769,7 +16575,7 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM90-NEXT: mov.b32 %r18, %r3;
-; SM90-NEXT: @%p1 bra $L__BB276_1;
+; SM90-NEXT: @%p1 bra $L__BB369_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: st.param.b16 [func_retval0], %r17;
@@ -13778,6 +16584,110 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
ret <2 x i8> %retval
}
+define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: nand_monotonic_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB370_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB370_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_monotonic_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB371_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB371_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: nand_acquire_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [nand_acquire_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b8 %r6, [nand_acquire_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM90-NEXT: $L__BB372_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
+; SM90-NEXT: @%p1 bra $L__BB372_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r15;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
+ ret <1 x i8> %retval
+}
+
define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM90-LABEL: nand_acquire_v2i8_global_cta(
; SM90: {
@@ -13798,7 +16708,7 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: shl.b32 %r8, %r7, %r1;
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: $L__BB277_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB373_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r18, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -13816,7 +16726,7 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM90-NEXT: mov.b32 %r18, %r3;
-; SM90-NEXT: @%p1 bra $L__BB277_1;
+; SM90-NEXT: @%p1 bra $L__BB373_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -13826,56 +16736,114 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
ret <2 x i8> %retval
}
-define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM90-LABEL: nand_release_v2i8_global_cta(
+define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: nand_acquire_v4i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<9>;
-; SM90-NEXT: .reg .b32 %r<19>;
-; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
-; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: and.b32 %r6, %r5, 3;
-; SM90-NEXT: shl.b32 %r1, %r6, 3;
-; SM90-NEXT: mov.b32 %r7, 65535;
-; SM90-NEXT: shl.b32 %r8, %r7, %r1;
-; SM90-NEXT: not.b32 %r2, %r8;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: $L__BB278_1: // %atomicrmw.start
+; SM90-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB374_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r9, %r18, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM90-NEXT: and.b32 %r11, %r10, %r4;
-; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM90-NEXT: shl.b32 %r14, %r13, %r1;
-; SM90-NEXT: and.b32 %r15, %r18, %r2;
-; SM90-NEXT: or.b32 %r16, %r15, %r14;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM90-NEXT: mov.b32 %r18, %r3;
-; SM90-NEXT: @%p1 bra $L__BB278_1;
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB374_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r17, %r3, %r1;
-; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
- ret <2 x i8> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
}
-define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM90-LABEL: nand_seq_cst_v2i8_global_cta(
+define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_acquire_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB375_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB375_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: nand_release_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [nand_release_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r6, [nand_release_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM90-NEXT: $L__BB376_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
+; SM90-NEXT: @%p1 bra $L__BB376_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
+; SM90-NEXT: st.param.b8 [func_retval0], %r15;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: nand_release_v2i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
; SM90-NEXT: .reg .b16 %rs<9>;
@@ -13883,10 +16851,10 @@ define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
; SM90-NEXT: and.b32 %r6, %r5, 3;
@@ -13895,7 +16863,7 @@ define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: shl.b32 %r8, %r7, %r1;
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: $L__BB279_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB377_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r18, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -13913,94 +16881,169 @@ define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM90-NEXT: mov.b32 %r18, %r3;
-; SM90-NEXT: @%p1 bra $L__BB279_1;
+; SM90-NEXT: @%p1 bra $L__BB377_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
-; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b16 [func_retval0], %r17;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
ret <2 x i8> %retval
}
-define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM90-LABEL: nand_monotonic_v4i8_global_cta(
+define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: nand_release_v4i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
; SM90-NEXT: .reg .b32 %r<6>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB280_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB378_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r5, %r2;
; SM90-NEXT: xor.b32 %r4, %r3, -1;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM90-NEXT: mov.b32 %r5, %r1;
-; SM90-NEXT: @%p1 bra $L__BB280_1;
+; SM90-NEXT: @%p1 bra $L__BB378_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
ret <4 x i8> %retval
}
-define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM90-LABEL: nand_acquire_v4i8_global_cta(
+define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_release_v8i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<6>;
-; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB281_1: // %atomicrmw.start
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB379_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: and.b32 %r3, %r5, %r2;
-; SM90-NEXT: xor.b32 %r4, %r3, -1;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
-; SM90-NEXT: mov.b32 %r5, %r1;
-; SM90-NEXT: @%p1 bra $L__BB281_1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB379_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: nand_seq_cst_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.b8 %r6, [nand_seq_cst_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM90-NEXT: $L__BB380_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
+; SM90-NEXT: @%p1 bra $L__BB380_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param.b8 [func_retval0], %r15;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
- ret <4 x i8> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
+ ret <1 x i8> %retval
}
-define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM90-LABEL: nand_release_v4i8_global_cta(
+define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: nand_seq_cst_v2i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<6>;
-; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB282_1: // %atomicrmw.start
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB381_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: and.b32 %r3, %r5, %r2;
-; SM90-NEXT: xor.b32 %r4, %r3, -1;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
-; SM90-NEXT: mov.b32 %r5, %r1;
-; SM90-NEXT: @%p1 bra $L__BB282_1;
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, %r4;
+; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB381_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
- ret <4 x i8> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
}
define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
@@ -14015,14 +17058,14 @@ define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
; SM90-NEXT: fence.sc.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB283_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB382_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r5, %r2;
; SM90-NEXT: xor.b32 %r4, %r3, -1;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM90-NEXT: mov.b32 %r5, %r1;
-; SM90-NEXT: @%p1 bra $L__BB283_1;
+; SM90-NEXT: @%p1 bra $L__BB382_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -14031,542 +17074,121 @@ define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
ret <4 x i8> %retval
}
-define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM90-LABEL: nand_monotonic_v8i8_global_cta(
+define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_seq_cst_v8i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
; SM90-NEXT: .reg .b32 %r<9>;
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: $L__BB284_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB383_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: and.b32 %r3, %r8, %r2;
-; SM90-NEXT: and.b32 %r4, %r7, %r1;
-; SM90-NEXT: xor.b32 %r5, %r4, -1;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NEXT: xor.b32 %r6, %r3, -1;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
-; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
-; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB284_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
- ret <8 x i8> %retval
-}
-
-define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM90-LABEL: nand_acquire_v8i8_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<9>;
-; SM90-NEXT: .reg .b64 %rd<12>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: $L__BB285_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: and.b32 %r3, %r8, %r2;
-; SM90-NEXT: and.b32 %r4, %r7, %r1;
-; SM90-NEXT: xor.b32 %r5, %r4, -1;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NEXT: xor.b32 %r6, %r3, -1;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
-; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
-; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB285_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
- ret <8 x i8> %retval
-}
-
-define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM90-LABEL: nand_release_v8i8_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<9>;
-; SM90-NEXT: .reg .b64 %rd<12>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: $L__BB286_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: and.b32 %r3, %r8, %r2;
-; SM90-NEXT: and.b32 %r4, %r7, %r1;
-; SM90-NEXT: xor.b32 %r5, %r4, -1;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NEXT: xor.b32 %r6, %r3, -1;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
-; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
-; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB286_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
- ret <8 x i8> %retval
-}
-
-define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM90-LABEL: nand_seq_cst_v8i8_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<9>;
-; SM90-NEXT: .reg .b64 %rd<12>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: $L__BB287_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: and.b32 %r3, %r8, %r2;
-; SM90-NEXT: and.b32 %r4, %r7, %r1;
-; SM90-NEXT: xor.b32 %r5, %r4, -1;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NEXT: xor.b32 %r6, %r3, -1;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
-; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
-; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB287_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
- ret <8 x i8> %retval
-}
-
-define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM90-LABEL: nand_monotonic_v2i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<7>;
-; SM90-NEXT: .reg .b64 %rd<13>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB288_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: and.b32 %r3, %r6, %r2;
-; SM90-NEXT: and.b32 %r4, %r5, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: not.b64 %rd11, %rd10;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM90-NEXT: @%p1 bra $L__BB288_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
- ret <2 x i32> %retval
-}
-
-define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM90-LABEL: nand_acquire_v2i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<7>;
-; SM90-NEXT: .reg .b64 %rd<13>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB289_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: and.b32 %r3, %r6, %r2;
-; SM90-NEXT: and.b32 %r4, %r5, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: not.b64 %rd11, %rd10;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM90-NEXT: @%p1 bra $L__BB289_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
- ret <2 x i32> %retval
-}
-
-define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM90-LABEL: nand_release_v2i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<7>;
-; SM90-NEXT: .reg .b64 %rd<13>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB290_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: and.b32 %r3, %r6, %r2;
-; SM90-NEXT: and.b32 %r4, %r5, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: not.b64 %rd11, %rd10;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM90-NEXT: @%p1 bra $L__BB290_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
- ret <2 x i32> %retval
-}
-
-define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM90-LABEL: nand_seq_cst_v2i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<7>;
-; SM90-NEXT: .reg .b64 %rd<13>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB291_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: and.b32 %r3, %r6, %r2;
-; SM90-NEXT: and.b32 %r4, %r5, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: not.b64 %rd11, %rd10;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM90-NEXT: @%p1 bra $L__BB291_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
- ret <2 x i32> %retval
-}
-
-define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM90-LABEL: nand_monotonic_v4i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<13>;
-; SM90-NEXT: .reg .b64 %rd<28>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
-; SM90-NEXT: mov.b64 %rd2, 0;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB292_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
-; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
-; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
-; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
-; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-NEXT: and.b32 %r5, %r12, %r4;
-; SM90-NEXT: and.b32 %r6, %r11, %r3;
-; SM90-NEXT: and.b32 %r7, %r10, %r2;
-; SM90-NEXT: and.b32 %r8, %r9, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
-; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
-; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-NEXT: not.b64 %rd17, %rd16;
-; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
-; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
-; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM90-NEXT: not.b64 %rd22, %rd21;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
-; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
-; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
-; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
-; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
-; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB292_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
- ret <4 x i32> %retval
-}
-
-define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM90-LABEL: nand_acquire_v4i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<13>;
-; SM90-NEXT: .reg .b64 %rd<28>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
-; SM90-NEXT: mov.b64 %rd2, 0;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB293_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
-; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
-; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
-; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
-; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-NEXT: and.b32 %r5, %r12, %r4;
-; SM90-NEXT: and.b32 %r6, %r11, %r3;
-; SM90-NEXT: and.b32 %r7, %r10, %r2;
-; SM90-NEXT: and.b32 %r8, %r9, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
-; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
-; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-NEXT: not.b64 %rd17, %rd16;
-; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
-; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
-; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM90-NEXT: not.b64 %rd22, %rd21;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
-; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
-; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
-; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
-; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
-; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB293_1;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB383_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
- ret <4 x i32> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
}
-define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM90-LABEL: nand_release_v4i32_global_cta(
+define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: nand_monotonic_v1i32_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<13>;
-; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: mov.b64 %rd2, 0;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB294_1: // %atomicrmw.start
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB384_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
-; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
-; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
-; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
-; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-NEXT: and.b32 %r5, %r12, %r4;
-; SM90-NEXT: and.b32 %r6, %r11, %r3;
-; SM90-NEXT: and.b32 %r7, %r10, %r2;
-; SM90-NEXT: and.b32 %r8, %r9, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
-; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
-; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-NEXT: not.b64 %rd17, %rd16;
-; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
-; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
-; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM90-NEXT: not.b64 %rd22, %rd21;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
-; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
-; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
-; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
-; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
-; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB294_1;
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: not.b32 %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p1 bra $L__BB384_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
- ret <4 x i32> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
+ ret <1 x i32> %retval
}
-define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM90-LABEL: nand_seq_cst_v4i32_global_cta(
+define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_monotonic_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB385_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB385_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_monotonic_v4i32_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
; SM90-NEXT: .reg .b32 %r<13>;
; SM90-NEXT: .reg .b64 %rd<28>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -14579,7 +17201,7 @@ define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB295_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB386_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
@@ -14618,12 +17240,11 @@ define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB295_1;
+; SM90-NEXT: @%p1 bra $L__BB386_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
ret <4 x i32> %retval
}
@@ -14650,7 +17271,7 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB296_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB387_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
@@ -14689,7 +17310,7 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB296_1;
+; SM90-NEXT: @%p1 bra $L__BB387_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd28, 0;
; SM90-NEXT: {
@@ -14703,7 +17324,7 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
-; SM90-NEXT: $L__BB296_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB387_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
@@ -14742,13 +17363,149 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
-; SM90-NEXT: @%p2 bra $L__BB296_3;
+; SM90-NEXT: @%p2 bra $L__BB387_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
- ret <8 x i32> %retval
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: nand_acquire_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB388_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: not.b32 %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p1 bra $L__BB388_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_acquire_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB389_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB389_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_acquire_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB390_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r12, %r4;
+; SM90-NEXT: and.b32 %r6, %r11, %r3;
+; SM90-NEXT: and.b32 %r7, %r10, %r2;
+; SM90-NEXT: and.b32 %r8, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB390_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
}
define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
@@ -14774,7 +17531,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB391_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
@@ -14813,7 +17570,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB297_1;
+; SM90-NEXT: @%p1 bra $L__BB391_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd28, 0;
; SM90-NEXT: {
@@ -14827,7 +17584,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
-; SM90-NEXT: $L__BB297_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB391_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
@@ -14866,7 +17623,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
-; SM90-NEXT: @%p2 bra $L__BB297_3;
+; SM90-NEXT: @%p2 bra $L__BB391_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -14876,6 +17633,142 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: nand_release_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [nand_release_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB392_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: not.b32 %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p1 bra $L__BB392_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_release_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB393_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB393_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_release_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB394_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r12, %r4;
+; SM90-NEXT: and.b32 %r6, %r11, %r3;
+; SM90-NEXT: and.b32 %r7, %r10, %r2;
+; SM90-NEXT: and.b32 %r8, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB394_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM90-LABEL: nand_release_v8i32_global_cta(
; SM90: {
@@ -14900,7 +17793,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB298_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB395_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
@@ -14939,7 +17832,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB298_1;
+; SM90-NEXT: @%p1 bra $L__BB395_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd28, 0;
; SM90-NEXT: {
@@ -14953,7 +17846,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
-; SM90-NEXT: $L__BB298_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB395_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
@@ -14992,7 +17885,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
-; SM90-NEXT: @%p2 bra $L__BB298_3;
+; SM90-NEXT: @%p2 bra $L__BB395_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
@@ -15001,6 +17894,145 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
ret <8 x i32> %retval
}
+define <1 x i32> @nand_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: nand_seq_cst_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.b32 %r1, [nand_seq_cst_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB396_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: not.b32 %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p1 bra $L__BB396_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_seq_cst_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB397_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB397_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_seq_cst_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB398_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r12, %r4;
+; SM90-NEXT: and.b32 %r6, %r11, %r3;
+; SM90-NEXT: and.b32 %r7, %r10, %r2;
+; SM90-NEXT: and.b32 %r8, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB398_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM90-LABEL: nand_seq_cst_v8i32_global_cta(
; SM90: {
@@ -15025,7 +18057,7 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB299_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB399_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
@@ -15064,7 +18096,7 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB299_1;
+; SM90-NEXT: @%p1 bra $L__BB399_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd28, 0;
; SM90-NEXT: {
@@ -15078,7 +18110,7 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
-; SM90-NEXT: $L__BB299_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB399_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
@@ -15117,7 +18149,7 @@ define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
-; SM90-NEXT: @%p2 bra $L__BB299_3;
+; SM90-NEXT: @%p2 bra $L__BB399_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw.py b/llvm/test/CodeGen/NVPTX/atomicrmw.py
index 238fe45bf45a0..bd2ee112749e7 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw.py
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw.py
@@ -9,6 +9,8 @@
ORDERINGS = ["monotonic", "acquire", "release", "acq_rel", "seq_cst"]
+VECTOR_SIZES = [1, 2, 4, 8]
+
INTEGER_OPERATIONS = [
"xchg",
"add",
@@ -29,6 +31,13 @@
FLOATING_POINT_OPERATIONS = ["fadd", "fsub", "fmin", "fmax", "fminimum", "fmaximum"]
+DATATYPE_SUFFIXES = {
+ "float": "f32",
+ "double": "f64",
+ "half": "f16",
+ "bfloat": "bf16",
+}
+
ADDRSPACE_NUM_TO_ADDRSPACE = {0: "generic", 1: "global", 3: "shared"}
atomicrmw_func = Template(
@@ -39,6 +48,14 @@
"""
)
+atomicrmw_elementwise_func = Template(
+ """define ${datatype} @${operation}_${ordering}_${datatype_name}_${addrspace}_${ptx_scope}(ptr${addrspace_cast} %addr, ${datatype} %val) {
+ %retval = atomicrmw elementwise ${operation} ptr ${addrspace_cast} %addr, ${datatype} %val syncscope(\"${llvm_scope}\") ${ordering}
+ ret $datatype %retval
+}
+"""
+)
+
run_statement = Template(
"""; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | FileCheck %s --check-prefix=SM${sm}
; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | %ptxas-verify -arch=sm_${sm} %}
@@ -53,6 +70,14 @@ def get_addrspace_cast(addrspace):
return " addrspace({})".format(str(addrspace))
+def get_vector_datatype(datatype, size):
+ return "<{} x {}>".format(size, datatype)
+
+
+def get_vector_datatype_name(datatype, size):
+ return "v{}{}".format(size, DATATYPE_SUFFIXES.get(datatype, datatype))
+
+
if __name__ == "__main__":
for sm, ptx in TEST_SM_ARCH_PAIRS:
# Slice 1: Keep addrspace, llvm_scope, ordering fixed, generate all possible operations and sizes
@@ -114,3 +139,47 @@ def get_addrspace_cast(addrspace):
),
file=fp,
)
+
+ with open("atomicrmw-elementwise-sm{}.ll".format(str(sm)), "w") as fp:
+ print(run_statement.substitute(sm=sm, ptx=ptx, ptxfp=ptx / 10.0), file=fp)
+
+ def print_elementwise_atomicrmw(operation, datatype, ordering, size):
+ print(
+ atomicrmw_elementwise_func.substitute(
+ operation=operation,
+ ordering=ordering,
+ datatype=get_vector_datatype(datatype, size),
+ datatype_name=get_vector_datatype_name(datatype, size),
+ addrspace=ADDRSPACE_NUM_TO_ADDRSPACE[addrspace],
+ ptx_scope=SCOPE_LLVM_TO_PTX[llvm_scope],
+ llvm_scope=llvm_scope,
+ addrspace_cast=get_addrspace_cast(addrspace),
+ ),
+ file=fp,
+ )
+
+ # Integer operations
+ addrspace, llvm_scope, ordering = 1, "block", "acq_rel"
+ for operation, datatype, size in product(
+ INTEGER_OPERATIONS, ["i8", "i16", "i32", "i64"], VECTOR_SIZES
+ ):
+ print_elementwise_atomicrmw(operation, datatype, ordering, size)
+
+ # Floating point operations
+ for datatype, operation, size in product(
+ ["float", "double", "half", "bfloat"],
+ FLOATING_POINT_OPERATIONS,
+ VECTOR_SIZES,
+ ):
+ print_elementwise_atomicrmw(operation, datatype, ordering, size)
+
+ # Slice 2: Keep addrspace, llvm_scope fixed, and generate all possible orderings for operations add and nand.
+ # add is natively supported for larger bitwidths, while nand is emulated always
+ addrspace, llvm_scope = 1, "block"
+ for operation, datatype, ordering, size in product(
+ ["add", "nand"], ["i8", "i32"], ORDERINGS, VECTOR_SIZES
+ ):
+ if addrspace == 1 and llvm_scope == "block" and ordering == "acq_rel":
+ # These are a part of Slice 1
+ continue
+ print_elementwise_atomicrmw(operation, datatype, ordering, size)
diff --git a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
index 7315a456f12c1..035f5021c946d 100644
--- a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
+++ b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
@@ -3,6 +3,21 @@
target triple = "nvptx64-nvidia-cuda"
+; A <1 x T> elementwise atomicrmw collapses directly to a scalar RMW and wraps
+; the old value back into a single-lane vector.
+define <1 x float> @fadd_v1f32_elementwise(ptr %addr, <1 x float> %val) {
+; CHECK-LABEL: @fadd_v1f32_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LANE_VAL:%.*]] = extractelement <1 x float> [[VAL:%.*]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR:%.*]], float [[LANE_VAL]] monotonic, align 4
+; CHECK-NEXT: [[LANE_OLD:%.*]] = insertelement <1 x float> poison, float [[TMP0]], i64 0
+; CHECK-NEXT: ret <1 x float> [[LANE_OLD]]
+;
+entry:
+ %old = atomicrmw elementwise fadd ptr %addr, <1 x float> %val monotonic
+ ret <1 x float> %old
+}
+
; Elementwise expansion: scalar f32 atomics are legal, so the vector is split
; into scalar atomicrmw instructions.
define <2 x float> @fadd_v2f32_elementwise(ptr %addr, <2 x float> %val) {
>From 94323f5d247b88779e5757aa98276386d1f274ad Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Sat, 9 May 2026 01:14:24 +0000
Subject: [PATCH 5/5] format
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 6fc2c316ccb81..92f6e35e3e3f9 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7559,7 +7559,7 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
auto *VecTy = cast<FixedVectorType>(AI->getType());
Type *LaneTy = VecTy->getElementType();
- // Collapse <1 x T> elementwise RMWs to scalar RMWs
+ // Collapse <1 x T> elementwise RMWs to scalar RMWs
if (VecTy->getNumElements() == 1)
return AtomicExpansionKind::Elementwise;
@@ -7650,8 +7650,8 @@ AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
STI.getMinCmpXchgSizeInBits())
return AtomicOrdering::Acquire;
if (auto *RI = dyn_cast<AtomicRMWInst>(I);
- RI && RI->getOrdering() == AtomicOrdering::SequentiallyConsistent &&
- shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::None)
+ RI && RI->getOrdering() == AtomicOrdering::SequentiallyConsistent &&
+ shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::None)
return AtomicOrdering::Acquire;
return AtomicOrdering::Monotonic;
@@ -7707,7 +7707,7 @@ Instruction *NVPTXTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
else {
AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(RI);
NeedsTrailingAquireFence = Expansion == AtomicExpansionKind::CmpXChg ||
- Expansion == AtomicExpansionKind::Elementwise;
+ Expansion == AtomicExpansionKind::Elementwise;
}
if (NeedsTrailingAquireFence)
More information about the llvm-commits
mailing list